2024-12-04 06:40:25,015 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 06:40:25,029 main DEBUG Took 0.012163 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 06:40:25,029 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 06:40:25,030 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 06:40:25,031 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 06:40:25,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,041 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 06:40:25,057 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,060 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,061 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,062 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,062 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,063 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,063 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,064 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,065 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,066 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,067 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,067 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,068 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,070 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,071 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 06:40:25,072 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,073 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 06:40:25,075 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 06:40:25,076 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 06:40:25,078 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 06:40:25,079 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 06:40:25,081 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 06:40:25,081 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 06:40:25,093 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 06:40:25,096 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 06:40:25,098 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 06:40:25,099 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 06:40:25,099 main DEBUG createAppenders(={Console}) 2024-12-04 06:40:25,100 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-04 06:40:25,101 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-04 06:40:25,101 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-04 06:40:25,102 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 06:40:25,102 main DEBUG OutputStream closed 2024-12-04 06:40:25,103 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 06:40:25,103 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 06:40:25,103 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-04 06:40:25,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 06:40:25,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 06:40:25,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 06:40:25,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 06:40:25,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 06:40:25,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 06:40:25,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 06:40:25,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 06:40:25,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 06:40:25,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 06:40:25,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 06:40:25,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 06:40:25,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 06:40:25,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 06:40:25,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 06:40:25,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 06:40:25,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 06:40:25,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 06:40:25,242 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 06:40:25,242 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-04 06:40:25,242 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 06:40:25,243 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-04T06:40:25,643 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c 2024-12-04 06:40:25,649 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 06:40:25,650 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T06:40:25,665 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-04T06:40:25,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T06:40:25,733 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96, deleteOnExit=true 2024-12-04T06:40:25,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T06:40:25,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/test.cache.data in system properties and HBase conf 2024-12-04T06:40:25,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T06:40:25,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.log.dir in system properties and HBase conf 2024-12-04T06:40:25,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T06:40:25,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T06:40:25,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T06:40:25,848 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T06:40:25,991 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T06:40:25,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T06:40:25,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T06:40:25,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T06:40:25,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T06:40:25,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T06:40:25,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T06:40:25,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T06:40:26,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T06:40:26,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T06:40:26,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/nfs.dump.dir in system properties and HBase conf 2024-12-04T06:40:26,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/java.io.tmpdir in system properties and HBase conf 2024-12-04T06:40:26,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T06:40:26,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T06:40:26,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T06:40:26,974 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T06:40:27,084 INFO [Time-limited test {}] log.Log(170): Logging initialized @2972ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T06:40:27,184 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T06:40:27,255 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T06:40:27,285 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T06:40:27,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T06:40:27,287 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T06:40:27,300 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T06:40:27,304 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.log.dir/,AVAILABLE} 2024-12-04T06:40:27,305 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T06:40:27,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/java.io.tmpdir/jetty-localhost-33269-hadoop-hdfs-3_4_1-tests_jar-_-any-11748890069959999651/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T06:40:27,569 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:33269} 2024-12-04T06:40:27,569 INFO [Time-limited test {}] server.Server(415): Started @3459ms 2024-12-04T06:40:28,010 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T06:40:28,018 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T06:40:28,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T06:40:28,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T06:40:28,019 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T06:40:28,020 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b40f5c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.log.dir/,AVAILABLE} 2024-12-04T06:40:28,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1118b2a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T06:40:28,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b60cd11{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/java.io.tmpdir/jetty-localhost-33065-hadoop-hdfs-3_4_1-tests_jar-_-any-7626579159177817065/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T06:40:28,163 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286a1f36{HTTP/1.1, (http/1.1)}{localhost:33065} 2024-12-04T06:40:28,163 INFO [Time-limited test {}] server.Server(415): Started @4053ms 2024-12-04T06:40:28,224 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T06:40:28,913 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data1/current/BP-1546864157-172.17.0.2-1733294426706/current, will proceed with Du for space computation calculation, 2024-12-04T06:40:28,913 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data2/current/BP-1546864157-172.17.0.2-1733294426706/current, will proceed with Du for space computation calculation, 2024-12-04T06:40:28,995 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T06:40:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8db685cfb37cd0 with lease ID 0x92df39e36f8d78bb: Processing first storage report for DS-adb50eb9-2644-44f1-ab94-6e6ec69c8275 from datanode DatanodeRegistration(127.0.0.1:38377, datanodeUuid=3fd41660-1e1d-4ccd-869d-fa052efa4727, infoPort=45223, infoSecurePort=0, ipcPort=43749, storageInfo=lv=-57;cid=testClusterID;nsid=631019187;c=1733294426706) 2024-12-04T06:40:29,057 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8db685cfb37cd0 with lease ID 0x92df39e36f8d78bb: from storage DS-adb50eb9-2644-44f1-ab94-6e6ec69c8275 node DatanodeRegistration(127.0.0.1:38377, datanodeUuid=3fd41660-1e1d-4ccd-869d-fa052efa4727, infoPort=45223, infoSecurePort=0, ipcPort=43749, storageInfo=lv=-57;cid=testClusterID;nsid=631019187;c=1733294426706), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-12-04T06:40:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8db685cfb37cd0 with lease ID 0x92df39e36f8d78bb: Processing first storage report for DS-d88984a4-6934-4f55-8ff7-4104c1180dc6 from datanode DatanodeRegistration(127.0.0.1:38377, datanodeUuid=3fd41660-1e1d-4ccd-869d-fa052efa4727, infoPort=45223, infoSecurePort=0, ipcPort=43749, storageInfo=lv=-57;cid=testClusterID;nsid=631019187;c=1733294426706) 2024-12-04T06:40:29,058 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8db685cfb37cd0 with lease ID 0x92df39e36f8d78bb: from storage DS-d88984a4-6934-4f55-8ff7-4104c1180dc6 node DatanodeRegistration(127.0.0.1:38377, datanodeUuid=3fd41660-1e1d-4ccd-869d-fa052efa4727, infoPort=45223, infoSecurePort=0, ipcPort=43749, storageInfo=lv=-57;cid=testClusterID;nsid=631019187;c=1733294426706), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T06:40:29,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c 2024-12-04T06:40:29,168 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/zookeeper_0, clientPort=53418, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T06:40:29,181 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53418 2024-12-04T06:40:29,195 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:29,198 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:29,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741825_1001 (size=7) 2024-12-04T06:40:29,856 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d with version=8 2024-12-04T06:40:29,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/hbase-staging 2024-12-04T06:40:29,962 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T06:40:30,226 INFO [Time-limited test {}] client.ConnectionUtils(128): master/607fd5c6574c:0 server-side Connection retries=45 2024-12-04T06:40:30,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,243 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,250 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T06:40:30,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T06:40:30,448 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T06:40:30,527 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T06:40:30,536 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T06:40:30,540 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T06:40:30,573 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1548 (auto-detected) 2024-12-04T06:40:30,574 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T06:40:30,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45619 2024-12-04T06:40:30,629 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45619 connecting to ZooKeeper ensemble=127.0.0.1:53418 2024-12-04T06:40:30,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456190x0, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T06:40:30,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45619-0x1017c33fe6b0000 connected 2024-12-04T06:40:30,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:30,718 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:30,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T06:40:30,734 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d, hbase.cluster.distributed=false 2024-12-04T06:40:30,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T06:40:30,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45619 2024-12-04T06:40:30,767 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45619 2024-12-04T06:40:30,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45619 2024-12-04T06:40:30,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45619 2024-12-04T06:40:30,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45619 2024-12-04T06:40:30,914 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/607fd5c6574c:0 server-side Connection retries=45 2024-12-04T06:40:30,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,917 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,918 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T06:40:30,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T06:40:30,918 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T06:40:30,921 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T06:40:30,923 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T06:40:30,924 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38257 2024-12-04T06:40:30,926 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38257 connecting to ZooKeeper ensemble=127.0.0.1:53418 2024-12-04T06:40:30,928 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:30,930 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:30,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382570x0, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T06:40:30,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38257-0x1017c33fe6b0001 connected 2024-12-04T06:40:30,940 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T06:40:30,944 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T06:40:30,955 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T06:40:30,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T06:40:30,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T06:40:30,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38257 2024-12-04T06:40:30,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38257 2024-12-04T06:40:30,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38257 2024-12-04T06:40:30,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38257 2024-12-04T06:40:30,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38257 2024-12-04T06:40:30,993 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;607fd5c6574c:45619 2024-12-04T06:40:30,994 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/607fd5c6574c,45619,1733294430018 2024-12-04T06:40:31,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T06:40:31,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T06:40:31,005 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/607fd5c6574c,45619,1733294430018 2024-12-04T06:40:31,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:31,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T06:40:31,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:31,029 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T06:40:31,031 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/607fd5c6574c,45619,1733294430018 from backup master directory 2024-12-04T06:40:31,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T06:40:31,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/607fd5c6574c,45619,1733294430018 2024-12-04T06:40:31,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T06:40:31,037 WARN [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T06:40:31,037 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=607fd5c6574c,45619,1733294430018 2024-12-04T06:40:31,039 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T06:40:31,041 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T06:40:31,108 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/hbase.id] with ID: 75f52202-11fd-4086-b847-f37b0601f8c4 2024-12-04T06:40:31,109 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/.tmp/hbase.id 2024-12-04T06:40:31,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741826_1002 (size=42) 2024-12-04T06:40:31,123 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/.tmp/hbase.id]:[hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/hbase.id] 2024-12-04T06:40:31,176 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T06:40:31,182 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T06:40:31,207 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-12-04T06:40:31,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:31,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:31,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741827_1003 (size=196) 2024-12-04T06:40:31,255 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:40:31,258 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T06:40:31,292 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:31,298 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T06:40:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741828_1004 (size=1189) 2024-12-04T06:40:31,361 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store 2024-12-04T06:40:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741829_1005 (size=34) 2024-12-04T06:40:31,791 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T06:40:31,796 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:31,797 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T06:40:31,798 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:40:31,798 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:40:31,800 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T06:40:31,800 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:40:31,801 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:40:31,802 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733294431797Disabling compacts and flushes for region at 1733294431797Disabling writes for close at 1733294431800 (+3 ms)Writing region close event to WAL at 1733294431801 (+1 ms)Closed at 1733294431801 2024-12-04T06:40:31,805 WARN [master/607fd5c6574c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/.initializing 2024-12-04T06:40:31,806 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/WALs/607fd5c6574c,45619,1733294430018 2024-12-04T06:40:31,816 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T06:40:31,834 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=607fd5c6574c%2C45619%2C1733294430018, suffix=, logDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/WALs/607fd5c6574c,45619,1733294430018, archiveDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/oldWALs, maxLogs=10 2024-12-04T06:40:31,867 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/WALs/607fd5c6574c,45619,1733294430018/607fd5c6574c%2C45619%2C1733294430018.1733294431840, exclude list is [], retry=0 2024-12-04T06:40:31,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-adb50eb9-2644-44f1-ab94-6e6ec69c8275,DISK] 2024-12-04T06:40:31,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T06:40:31,968 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/WALs/607fd5c6574c,45619,1733294430018/607fd5c6574c%2C45619%2C1733294430018.1733294431840 2024-12-04T06:40:31,969 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45223:45223)] 2024-12-04T06:40:31,970 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:40:31,970 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:31,974 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:31,976 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T06:40:32,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:32,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T06:40:32,063 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:32,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T06:40:32,067 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:32,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T06:40:32,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:32,072 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,076 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,077 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,082 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,083 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,086 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T06:40:32,090 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T06:40:32,095 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:40:32,096 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67127597, jitterRate=2.7914345264434814E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T06:40:32,102 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733294431994Initializing all the Stores at 1733294431997 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294431997Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294431998 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294431999 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294431999Cleaning up temporary data from old regions at 1733294432083 (+84 ms)Region opened successfully at 1733294432102 (+19 ms) 2024-12-04T06:40:32,103 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T06:40:32,140 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78de2f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=607fd5c6574c/172.17.0.2:0 2024-12-04T06:40:32,174 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T06:40:32,186 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T06:40:32,186 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T06:40:32,189 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T06:40:32,190 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T06:40:32,195 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T06:40:32,195 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T06:40:32,229 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T06:40:32,239 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T06:40:32,241 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T06:40:32,243 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T06:40:32,245 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T06:40:32,247 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T06:40:32,249 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T06:40:32,253 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T06:40:32,255 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T06:40:32,257 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T06:40:32,260 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T06:40:32,281 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T06:40:32,283 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T06:40:32,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T06:40:32,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T06:40:32,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,291 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=607fd5c6574c,45619,1733294430018, sessionid=0x1017c33fe6b0000, setting cluster-up flag (Was=false) 2024-12-04T06:40:32,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,312 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T06:40:32,313 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=607fd5c6574c,45619,1733294430018 2024-12-04T06:40:32,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,329 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T06:40:32,330 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=607fd5c6574c,45619,1733294430018 2024-12-04T06:40:32,336 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T06:40:32,375 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(746): ClusterId : 75f52202-11fd-4086-b847-f37b0601f8c4 2024-12-04T06:40:32,378 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T06:40:32,384 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T06:40:32,384 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T06:40:32,387 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T06:40:32,388 DEBUG [RS:0;607fd5c6574c:38257 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@877422e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=607fd5c6574c/172.17.0.2:0 2024-12-04T06:40:32,404 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;607fd5c6574c:38257 2024-12-04T06:40:32,407 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T06:40:32,407 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T06:40:32,407 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T06:40:32,410 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(2659): reportForDuty to master=607fd5c6574c,45619,1733294430018 with port=38257, startcode=1733294430862 2024-12-04T06:40:32,415 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T06:40:32,424 DEBUG [RS:0;607fd5c6574c:38257 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T06:40:32,426 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T06:40:32,433 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T06:40:32,438 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 607fd5c6574c,45619,1733294430018 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T06:40:32,447 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/607fd5c6574c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T06:40:32,448 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/607fd5c6574c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T06:40:32,449 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/607fd5c6574c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T06:40:32,449 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/607fd5c6574c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T06:40:32,449 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/607fd5c6574c:0, corePoolSize=10, maxPoolSize=10 2024-12-04T06:40:32,450 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,450 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/607fd5c6574c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T06:40:32,450 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,460 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T06:40:32,461 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T06:40:32,465 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733294462465 2024-12-04T06:40:32,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58727, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T06:40:32,467 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T06:40:32,468 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T06:40:32,468 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,468 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T06:40:32,473 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45619 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:32,474 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T06:40:32,475 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T06:40:32,475 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T06:40:32,475 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T06:40:32,479 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,485 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T06:40:32,486 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T06:40:32,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741831_1007 (size=1321) 2024-12-04T06:40:32,487 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T06:40:32,490 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T06:40:32,490 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T06:40:32,497 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.large.0-1733294432496,5,FailOnTimeoutGroup] 2024-12-04T06:40:32,497 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T06:40:32,498 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:40:32,500 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.small.0-1733294432497,5,FailOnTimeoutGroup] 2024-12-04T06:40:32,500 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,501 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T06:40:32,502 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,502 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,505 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T06:40:32,505 WARN [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T06:40:32,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741832_1008 (size=32) 2024-12-04T06:40:32,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:32,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T06:40:32,513 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T06:40:32,513 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:32,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T06:40:32,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T06:40:32,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:32,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T06:40:32,520 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T06:40:32,520 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:32,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T06:40:32,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T06:40:32,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:32,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:32,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T06:40:32,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740 2024-12-04T06:40:32,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740 2024-12-04T06:40:32,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T06:40:32,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T06:40:32,529 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:40:32,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T06:40:32,535 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:40:32,536 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59936384, jitterRate=-0.10687828063964844}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:40:32,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733294432509Initializing all the Stores at 1733294432510 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294432510Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294432511 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294432511Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294432511Cleaning up temporary data from old regions at 1733294432529 (+18 ms)Region opened successfully at 1733294432538 (+9 ms) 2024-12-04T06:40:32,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T06:40:32,539 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T06:40:32,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T06:40:32,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T06:40:32,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T06:40:32,540 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T06:40:32,540 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733294432539Disabling compacts and flushes for region at 1733294432539Disabling writes for close at 1733294432539Writing region close event to WAL at 1733294432540 (+1 ms)Closed at 1733294432540 2024-12-04T06:40:32,544 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T06:40:32,544 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T06:40:32,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T06:40:32,559 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T06:40:32,562 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T06:40:32,606 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(2659): reportForDuty to master=607fd5c6574c,45619,1733294430018 with port=38257, startcode=1733294430862 2024-12-04T06:40:32,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45619 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,611 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45619 {}] master.ServerManager(517): Registering regionserver=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,620 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:40:32,620 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38371 2024-12-04T06:40:32,620 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T06:40:32,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T06:40:32,625 DEBUG [RS:0;607fd5c6574c:38257 {}] zookeeper.ZKUtil(111): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,625 WARN [RS:0;607fd5c6574c:38257 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T06:40:32,626 INFO [RS:0;607fd5c6574c:38257 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T06:40:32,626 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,628 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [607fd5c6574c,38257,1733294430862] 2024-12-04T06:40:32,656 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T06:40:32,671 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T06:40:32,677 INFO [RS:0;607fd5c6574c:38257 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T06:40:32,677 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,679 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T06:40:32,687 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T06:40:32,689 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,689 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,689 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,690 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,690 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,690 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,690 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/607fd5c6574c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T06:40:32,690 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,691 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,691 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,691 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,691 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,692 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/607fd5c6574c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T06:40:32,692 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/607fd5c6574c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T06:40:32,692 DEBUG [RS:0;607fd5c6574c:38257 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T06:40:32,693 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,693 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,694 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,694 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,694 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,694 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,38257,1733294430862-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T06:40:32,713 WARN [607fd5c6574c:45619 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T06:40:32,716 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T06:40:32,719 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,38257,1733294430862-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,719 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,720 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.Replication(171): 607fd5c6574c,38257,1733294430862 started 2024-12-04T06:40:32,745 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:32,746 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1482): Serving as 607fd5c6574c,38257,1733294430862, RpcServer on 607fd5c6574c/172.17.0.2:38257, sessionid=0x1017c33fe6b0001 2024-12-04T06:40:32,747 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T06:40:32,747 DEBUG [RS:0;607fd5c6574c:38257 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,747 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '607fd5c6574c,38257,1733294430862' 2024-12-04T06:40:32,747 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T06:40:32,748 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T06:40:32,749 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T06:40:32,749 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T06:40:32,749 DEBUG [RS:0;607fd5c6574c:38257 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,750 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '607fd5c6574c,38257,1733294430862' 2024-12-04T06:40:32,750 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T06:40:32,750 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T06:40:32,751 DEBUG [RS:0;607fd5c6574c:38257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T06:40:32,751 INFO [RS:0;607fd5c6574c:38257 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T06:40:32,751 INFO [RS:0;607fd5c6574c:38257 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T06:40:32,857 INFO [RS:0;607fd5c6574c:38257 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T06:40:32,860 INFO [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=607fd5c6574c%2C38257%2C1733294430862, suffix=, logDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862, archiveDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/oldWALs, maxLogs=32 2024-12-04T06:40:32,877 DEBUG [RS:0;607fd5c6574c:38257 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862/607fd5c6574c%2C38257%2C1733294430862.1733294432862, exclude list is [], retry=0 2024-12-04T06:40:32,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-adb50eb9-2644-44f1-ab94-6e6ec69c8275,DISK] 2024-12-04T06:40:32,884 INFO [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862/607fd5c6574c%2C38257%2C1733294430862.1733294432862 2024-12-04T06:40:32,888 DEBUG [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45223:45223)] 2024-12-04T06:40:32,966 DEBUG [607fd5c6574c:45619 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T06:40:32,978 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:32,985 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 607fd5c6574c,38257,1733294430862, state=OPENING 2024-12-04T06:40:32,992 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T06:40:32,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:40:32,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T06:40:32,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T06:40:32,996 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T06:40:32,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:40:33,175 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T06:40:33,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T06:40:33,189 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T06:40:33,189 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T06:40:33,190 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T06:40:33,193 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=607fd5c6574c%2C38257%2C1733294430862.meta, suffix=.meta, logDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862, archiveDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/oldWALs, maxLogs=32 2024-12-04T06:40:33,209 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862/607fd5c6574c%2C38257%2C1733294430862.meta.1733294433194.meta, exclude list is [], retry=0 2024-12-04T06:40:33,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-adb50eb9-2644-44f1-ab94-6e6ec69c8275,DISK] 2024-12-04T06:40:33,215 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/WALs/607fd5c6574c,38257,1733294430862/607fd5c6574c%2C38257%2C1733294430862.meta.1733294433194.meta 2024-12-04T06:40:33,215 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45223:45223)] 2024-12-04T06:40:33,216 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:40:33,218 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T06:40:33,221 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T06:40:33,227 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T06:40:33,233 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T06:40:33,234 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:33,234 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T06:40:33,234 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T06:40:33,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T06:40:33,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T06:40:33,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:33,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:33,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T06:40:33,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T06:40:33,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:33,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:33,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T06:40:33,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T06:40:33,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:33,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:33,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T06:40:33,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T06:40:33,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:33,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T06:40:33,249 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T06:40:33,250 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740 2024-12-04T06:40:33,253 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740 2024-12-04T06:40:33,255 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T06:40:33,255 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T06:40:33,256 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:40:33,258 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T06:40:33,259 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62984164, jitterRate=-0.06146281957626343}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:40:33,260 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T06:40:33,261 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733294433235Writing region info on filesystem at 1733294433235Initializing all the Stores at 1733294433237 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294433237Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294433238 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294433238Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733294433238Cleaning up temporary data from old regions at 1733294433255 (+17 ms)Running coprocessor post-open hooks at 1733294433260 (+5 ms)Region opened successfully at 1733294433261 (+1 ms) 2024-12-04T06:40:33,268 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733294433166 2024-12-04T06:40:33,279 DEBUG [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T06:40:33,280 INFO [RS_OPEN_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T06:40:33,281 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:33,283 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 607fd5c6574c,38257,1733294430862, state=OPEN 2024-12-04T06:40:33,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T06:40:33,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T06:40:33,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T06:40:33,292 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T06:40:33,292 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:33,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T06:40:33,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=607fd5c6574c,38257,1733294430862 in 295 msec 2024-12-04T06:40:33,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T06:40:33,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 748 msec 2024-12-04T06:40:33,305 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T06:40:33,305 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T06:40:33,326 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:33,327 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:33,348 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:33,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59887, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:33,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0030 sec 2024-12-04T06:40:33,371 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733294433371, completionTime=-1 2024-12-04T06:40:33,373 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T06:40:33,374 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T06:40:33,401 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T06:40:33,401 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733294493401 2024-12-04T06:40:33,401 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733294553401 2024-12-04T06:40:33,401 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-04T06:40:33,403 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,404 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,404 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,405 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-607fd5c6574c:45619, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,406 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,407 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,413 DEBUG [master/607fd5c6574c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T06:40:33,439 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.402sec 2024-12-04T06:40:33,440 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T06:40:33,441 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T06:40:33,442 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T06:40:33,443 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T06:40:33,443 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T06:40:33,444 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T06:40:33,444 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T06:40:33,452 DEBUG [master/607fd5c6574c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T06:40:33,453 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T06:40:33,453 INFO [master/607fd5c6574c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=607fd5c6574c,45619,1733294430018-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T06:40:33,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a820c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:33,488 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T06:40:33,488 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T06:40:33,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:33,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:33,502 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437ec111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:33,510 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:33,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a052f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:33,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:33,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:33,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=607fd5c6574c,45619,1733294430018 2024-12-04T06:40:33,557 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=7279 2024-12-04T06:40:33,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:40:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:40:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e414e47 2024-12-04T06:40:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:40:33,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:40:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:40:33,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:40:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:40:33,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:40:33,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-04T06:40:33,626 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:33,629 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:40:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T06:40:33,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741835_1011 (size=1061) 2024-12-04T06:40:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T06:40:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T06:40:34,044 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:40:34,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741836_1012 (size=53) 2024-12-04T06:40:34,052 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:34,052 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c8a5d67afecdb7afe2ddb08e0bd45184, disabling compactions & flushes 2024-12-04T06:40:34,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. after waiting 0 ms 2024-12-04T06:40:34,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8a5d67afecdb7afe2ddb08e0bd45184: Waiting for close lock at 1733294434052Disabling compacts and flushes for region at 1733294434052Disabling writes for close at 1733294434053 (+1 ms)Writing region close event to WAL at 1733294434053Closed at 1733294434053 2024-12-04T06:40:34,055 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:40:34,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294434055"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294434055"}]},"ts":"1733294434055"} 2024-12-04T06:40:34,065 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:40:34,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:40:34,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294434067"}]},"ts":"1733294434067"} 2024-12-04T06:40:34,073 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:40:34,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, ASSIGN}] 2024-12-04T06:40:34,077 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, ASSIGN 2024-12-04T06:40:34,079 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:40:34,230 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8a5d67afecdb7afe2ddb08e0bd45184, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:34,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, ASSIGN because future has completed 2024-12-04T06:40:34,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:40:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T06:40:34,396 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,396 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:40:34,397 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,397 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:40:34,397 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,397 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,399 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,402 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:40:34,403 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8a5d67afecdb7afe2ddb08e0bd45184 columnFamilyName A 2024-12-04T06:40:34,403 DEBUG [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:34,404 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(327): Store=c8a5d67afecdb7afe2ddb08e0bd45184/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:34,404 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,406 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:40:34,406 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8a5d67afecdb7afe2ddb08e0bd45184 columnFamilyName B 2024-12-04T06:40:34,406 DEBUG [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:34,407 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(327): Store=c8a5d67afecdb7afe2ddb08e0bd45184/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:34,407 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,409 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:40:34,409 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8a5d67afecdb7afe2ddb08e0bd45184 columnFamilyName C 2024-12-04T06:40:34,409 DEBUG [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:40:34,410 INFO [StoreOpener-c8a5d67afecdb7afe2ddb08e0bd45184-1 {}] regionserver.HStore(327): Store=c8a5d67afecdb7afe2ddb08e0bd45184/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:40:34,410 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,410 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,411 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,412 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,414 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,414 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,415 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:40:34,417 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,420 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:40:34,421 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c8a5d67afecdb7afe2ddb08e0bd45184; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70093911, jitterRate=0.044480666518211365}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:40:34,421 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,422 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c8a5d67afecdb7afe2ddb08e0bd45184: Running coprocessor pre-open hook at 1733294434397Writing region info on filesystem at 1733294434397Initializing all the Stores at 1733294434399 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294434399Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294434399Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294434399Cleaning up temporary data from old regions at 1733294434414 (+15 ms)Running coprocessor post-open hooks at 1733294434421 (+7 ms)Region opened successfully at 1733294434422 (+1 ms) 2024-12-04T06:40:34,424 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., pid=6, masterSystemTime=1733294434389 2024-12-04T06:40:34,427 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,427 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:34,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8a5d67afecdb7afe2ddb08e0bd45184, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:34,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:40:34,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T06:40:34,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 in 199 msec 2024-12-04T06:40:34,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T06:40:34,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, ASSIGN in 363 msec 2024-12-04T06:40:34,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:40:34,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294434443"}]},"ts":"1733294434443"} 2024-12-04T06:40:34,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:40:34,447 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:40:34,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 834 msec 2024-12-04T06:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T06:40:34,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:34,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b79cc89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,778 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,783 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,789 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:40:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:40:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-12-04T06:40:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:40:34,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:40:34,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2090f191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a862833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,807 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4d5de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,813 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656f7adc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c642253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,828 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fcb8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,831 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f667ed6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62941f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1bc719, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d93931d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c15e47b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,841 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c391c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311faa9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:40:34,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:40:34,850 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:40:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:40:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:40:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71258907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:40:34,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:40:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:34,852 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:40:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cfd059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:40:34,858 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,858 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,858 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,858 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,860 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,860 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,862 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-04T06:40:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,872 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,874 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:34,874 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,886 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:40:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:40:34,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:40:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:40:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:34,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:40:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:35,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7722267f671b4a17805670bbd989b72e is 50, key is test_row_0/A:col10/1733294434938/Put/seqid=0 2024-12-04T06:40:35,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741837_1013 (size=9657) 2024-12-04T06:40:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294495084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733294435002, max timestamp=1733294435127Num uniques 859; 2024-12-04T06:40:35,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294435002, max timestamp=1733294435127Num uniques 859; 2024-12-04T06:40:35,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:40:35,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:40:35,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294435002, max timestamp=1733294435127Num uniques 859; 2024-12-04T06:40:35,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:40:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294495097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:35,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294495245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294495251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294495253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294495298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294495299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294495368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294495368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294495369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7722267f671b4a17805670bbd989b72e 2024-12-04T06:40:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294495508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294495517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294495578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294495587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294495588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:35,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:35,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:35,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/3ce1f65ae7584d10bd4e18b65cec8cff is 50, key is test_row_0/B:col10/1733294434938/Put/seqid=0 2024-12-04T06:40:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741838_1014 (size=9657) 2024-12-04T06:40:35,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/3ce1f65ae7584d10bd4e18b65cec8cff 2024-12-04T06:40:35,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/b022948605da4521b7953fdd2b0d42ad is 50, key is test_row_0/C:col10/1733294434938/Put/seqid=0 2024-12-04T06:40:35,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741839_1015 (size=9657) 2024-12-04T06:40:35,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/b022948605da4521b7953fdd2b0d42ad 2024-12-04T06:40:35,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7722267f671b4a17805670bbd989b72e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e 2024-12-04T06:40:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e, entries=100, sequenceid=13, filesize=9.4 K 2024-12-04T06:40:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/3ce1f65ae7584d10bd4e18b65cec8cff as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff 2024-12-04T06:40:35,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff, entries=100, sequenceid=13, filesize=9.4 K 2024-12-04T06:40:35,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/b022948605da4521b7953fdd2b0d42ad as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad 2024-12-04T06:40:35,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad, entries=100, sequenceid=13, filesize=9.4 K 2024-12-04T06:40:35,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c8a5d67afecdb7afe2ddb08e0bd45184 in 835ms, sequenceid=13, compaction requested=false 2024-12-04T06:40:35,794 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T06:40:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:35,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T06:40:35,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:35,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:35,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:35,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:35,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/80de97402b7e4e3ab1551bb099956ac1 is 50, key is test_row_0/A:col10/1733294435247/Put/seqid=0 2024-12-04T06:40:35,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741840_1016 (size=14341) 2024-12-04T06:40:35,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/80de97402b7e4e3ab1551bb099956ac1 2024-12-04T06:40:35,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4362678d468140c09821bf6f5efaf895 is 50, key is test_row_0/B:col10/1733294435247/Put/seqid=0 2024-12-04T06:40:35,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741841_1017 (size=12001) 2024-12-04T06:40:35,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4362678d468140c09821bf6f5efaf895 2024-12-04T06:40:35,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/dc8d85c360a145cb8b54ea1fb0d15831 is 50, key is test_row_0/C:col10/1733294435247/Put/seqid=0 2024-12-04T06:40:35,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741842_1018 (size=12001) 2024-12-04T06:40:36,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/dc8d85c360a145cb8b54ea1fb0d15831 2024-12-04T06:40:36,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294495944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294495953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/80de97402b7e4e3ab1551bb099956ac1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1 2024-12-04T06:40:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:36,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1, entries=200, sequenceid=48, filesize=14.0 K 2024-12-04T06:40:36,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4362678d468140c09821bf6f5efaf895 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895 2024-12-04T06:40:36,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:36,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:36,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T06:40:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/dc8d85c360a145cb8b54ea1fb0d15831 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831 2024-12-04T06:40:36,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831, entries=150, sequenceid=48, filesize=11.7 K 2024-12-04T06:40:36,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=60.38 KB/61830 for c8a5d67afecdb7afe2ddb08e0bd45184 in 212ms, sequenceid=48, compaction requested=false 2024-12-04T06:40:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:36,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ba77019c9a854d958baeeb4263e6de75 is 50, key is test_row_0/A:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741843_1019 (size=14341) 2024-12-04T06:40:36,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ba77019c9a854d958baeeb4263e6de75 2024-12-04T06:40:36,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/67a82d5b0b23402589f71b0474bd60a6 is 50, key is test_row_0/B:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:36,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294496190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294496194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741844_1020 (size=12001) 2024-12-04T06:40:36,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/67a82d5b0b23402589f71b0474bd60a6 2024-12-04T06:40:36,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e36bddb088614384a9c4b01cf16c9591 is 50, key is test_row_0/C:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741845_1021 (size=12001) 2024-12-04T06:40:36,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e36bddb088614384a9c4b01cf16c9591 2024-12-04T06:40:36,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ba77019c9a854d958baeeb4263e6de75 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75 2024-12-04T06:40:36,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75, entries=200, sequenceid=61, filesize=14.0 K 2024-12-04T06:40:36,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/67a82d5b0b23402589f71b0474bd60a6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6 2024-12-04T06:40:36,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T06:40:36,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e36bddb088614384a9c4b01cf16c9591 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591 2024-12-04T06:40:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294496308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294496309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591, entries=150, sequenceid=61, filesize=11.7 K 2024-12-04T06:40:36,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c8a5d67afecdb7afe2ddb08e0bd45184 in 203ms, sequenceid=61, compaction requested=true 2024-12-04T06:40:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:36,326 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:36,326 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:36,330 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:36,330 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:36,332 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:36,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:36,332 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,333 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=32.9 K 2024-12-04T06:40:36,334 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,334 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=37.4 K 2024-12-04T06:40:36,334 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce1f65ae7584d10bd4e18b65cec8cff, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294434938 2024-12-04T06:40:36,339 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7722267f671b4a17805670bbd989b72e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294434938 2024-12-04T06:40:36,339 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4362678d468140c09821bf6f5efaf895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733294435207 2024-12-04T06:40:36,340 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80de97402b7e4e3ab1551bb099956ac1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733294435207 2024-12-04T06:40:36,340 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a82d5b0b23402589f71b0474bd60a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:36,342 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba77019c9a854d958baeeb4263e6de75, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:36,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-04T06:40:36,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,349 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:40:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:36,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/274b3e7cf26d4c99bb8b88c4aef6488d is 50, key is test_row_0/A:col10/1733294436188/Put/seqid=0 2024-12-04T06:40:36,383 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:36,385 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/a55795bb1224449fae1fa2b7b7bd55e6 is 50, key is test_row_0/B:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,386 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:36,387 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/d261b12f97a54ba5a94db08292b3553a is 50, key is test_row_0/A:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741847_1023 (size=12104) 2024-12-04T06:40:36,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741846_1022 (size=12001) 2024-12-04T06:40:36,416 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/274b3e7cf26d4c99bb8b88c4aef6488d 2024-12-04T06:40:36,431 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/a55795bb1224449fae1fa2b7b7bd55e6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/a55795bb1224449fae1fa2b7b7bd55e6 2024-12-04T06:40:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741848_1024 (size=12104) 2024-12-04T06:40:36,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2690b41d255a460cb1c27d4ff68feeda is 50, key is test_row_0/B:col10/1733294436188/Put/seqid=0 2024-12-04T06:40:36,461 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into a55795bb1224449fae1fa2b7b7bd55e6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:36,461 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,464 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294436326; duration=0sec 2024-12-04T06:40:36,464 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:36,464 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:36,465 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:36,467 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:36,467 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:36,467 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,468 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=32.9 K 2024-12-04T06:40:36,469 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b022948605da4521b7953fdd2b0d42ad, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294434938 2024-12-04T06:40:36,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8d85c360a145cb8b54ea1fb0d15831, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733294435207 2024-12-04T06:40:36,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e36bddb088614384a9c4b01cf16c9591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:36,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741849_1025 (size=12001) 2024-12-04T06:40:36,502 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:36,503 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/5243debfd76d45bdac14b20066af3cfa is 50, key is test_row_0/C:col10/1733294435981/Put/seqid=0 2024-12-04T06:40:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:36,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741850_1026 (size=12104) 2024-12-04T06:40:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,553 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/5243debfd76d45bdac14b20066af3cfa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/5243debfd76d45bdac14b20066af3cfa 2024-12-04T06:40:36,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294496538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294496555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,569 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 5243debfd76d45bdac14b20066af3cfa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:36,569 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,569 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294436326; duration=0sec 2024-12-04T06:40:36,570 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:36,570 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294496658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294496669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,862 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/d261b12f97a54ba5a94db08292b3553a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/d261b12f97a54ba5a94db08292b3553a 2024-12-04T06:40:36,881 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into d261b12f97a54ba5a94db08292b3553a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:36,881 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,881 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294436323; duration=0sec 2024-12-04T06:40:36,881 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:36,882 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:36,884 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2690b41d255a460cb1c27d4ff68feeda 2024-12-04T06:40:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294496878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294496878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294496878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294496888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294496888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:36,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/94116d2ff86443589a5ed6cf6a77b527 is 50, key is test_row_0/C:col10/1733294436188/Put/seqid=0 2024-12-04T06:40:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741851_1027 (size=12001) 2024-12-04T06:40:36,927 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/94116d2ff86443589a5ed6cf6a77b527 2024-12-04T06:40:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/274b3e7cf26d4c99bb8b88c4aef6488d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d 2024-12-04T06:40:36,954 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d, entries=150, sequenceid=81, filesize=11.7 K 2024-12-04T06:40:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2690b41d255a460cb1c27d4ff68feeda as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda 2024-12-04T06:40:36,969 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda, entries=150, sequenceid=81, filesize=11.7 K 2024-12-04T06:40:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/94116d2ff86443589a5ed6cf6a77b527 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527 2024-12-04T06:40:36,987 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527, entries=150, sequenceid=81, filesize=11.7 K 2024-12-04T06:40:36,991 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c8a5d67afecdb7afe2ddb08e0bd45184 in 642ms, sequenceid=81, compaction requested=false 2024-12-04T06:40:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T06:40:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T06:40:37,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T06:40:37,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-12-04T06:40:37,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1370 sec 2024-12-04T06:40:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T06:40:37,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:37,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-04T06:40:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T06:40:37,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:37,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:37,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T06:40:37,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-04T06:40:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:37,194 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:37,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:37,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:37,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/a4b33a1c823446a9b4ccce7505735dde is 50, key is test_row_0/A:col10/1733294436550/Put/seqid=0 2024-12-04T06:40:37,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741852_1028 (size=12001) 2024-12-04T06:40:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294497246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294497249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294497250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294497252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294497253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T06:40:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294497357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294497368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294497368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294497368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294497368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294497567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294497578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294497578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294497578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294497587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,634 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/a4b33a1c823446a9b4ccce7505735dde 2024-12-04T06:40:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T06:40:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8c6643705d1b4ee698f9cc65c65df5cb is 50, key is test_row_0/B:col10/1733294436550/Put/seqid=0 2024-12-04T06:40:37,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741853_1029 (size=12001) 2024-12-04T06:40:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8c6643705d1b4ee698f9cc65c65df5cb 2024-12-04T06:40:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a01ff9af9b084ccbb85b0d795f5ea588 is 50, key is test_row_0/C:col10/1733294436550/Put/seqid=0 2024-12-04T06:40:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741854_1030 (size=12001) 2024-12-04T06:40:37,725 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a01ff9af9b084ccbb85b0d795f5ea588 2024-12-04T06:40:37,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/a4b33a1c823446a9b4ccce7505735dde as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde 2024-12-04T06:40:37,760 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T06:40:37,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8c6643705d1b4ee698f9cc65c65df5cb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb 2024-12-04T06:40:37,789 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T06:40:37,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a01ff9af9b084ccbb85b0d795f5ea588 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588 2024-12-04T06:40:37,812 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T06:40:37,814 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c8a5d67afecdb7afe2ddb08e0bd45184 in 620ms, sequenceid=97, compaction requested=true 2024-12-04T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:37,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T06:40:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T06:40:37,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T06:40:37,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 782 msec 2024-12-04T06:40:37,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 795 msec 2024-12-04T06:40:37,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294437242, max timestamp=1733294437884Num uniques 859; 2024-12-04T06:40:37,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:40:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:37,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-04T06:40:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:37,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/dc063ccbd9e54afe9a4a1fb88bf53430 is 50, key is test_row_0/A:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:37,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741855_1031 (size=16681) 2024-12-04T06:40:37,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/dc063ccbd9e54afe9a4a1fb88bf53430 2024-12-04T06:40:37,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/148e839daa6a469e836a30b6ee01cced is 50, key is test_row_0/B:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294497942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294497950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294497953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294497954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294497955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:37,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741856_1032 (size=12001) 2024-12-04T06:40:37,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/148e839daa6a469e836a30b6ee01cced 2024-12-04T06:40:38,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/85459510132f4047b388685bc5ec2ec0 is 50, key is test_row_0/C:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:38,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294498067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741857_1033 (size=12001) 2024-12-04T06:40:38,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294498077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294498077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294498078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294498078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T06:40:38,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:38,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-04T06:40:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:38,197 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:38,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:38,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294498287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294498296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294498297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294498297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294498297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:38,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/85459510132f4047b388685bc5ec2ec0 2024-12-04T06:40:38,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/dc063ccbd9e54afe9a4a1fb88bf53430 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430 2024-12-04T06:40:38,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430, entries=250, sequenceid=120, filesize=16.3 K 2024-12-04T06:40:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/148e839daa6a469e836a30b6ee01cced as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced 2024-12-04T06:40:38,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced, entries=150, sequenceid=120, filesize=11.7 K 2024-12-04T06:40:38,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:38,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/85459510132f4047b388685bc5ec2ec0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0 2024-12-04T06:40:38,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0, entries=150, sequenceid=120, filesize=11.7 K 2024-12-04T06:40:38,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~301.91 KB/309152, currentSize=73.80 KB/75570 for c8a5d67afecdb7afe2ddb08e0bd45184 in 635ms, sequenceid=120, compaction requested=true 2024-12-04T06:40:38,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:38,532 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:38,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T06:40:38,532 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:38,535 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:38,535 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:38,535 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:38,535 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,536 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:38,536 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/d261b12f97a54ba5a94db08292b3553a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=51.5 K 2024-12-04T06:40:38,536 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,536 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/a55795bb1224449fae1fa2b7b7bd55e6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=47.0 K 2024-12-04T06:40:38,537 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d261b12f97a54ba5a94db08292b3553a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:38,538 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274b3e7cf26d4c99bb8b88c4aef6488d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733294436162 2024-12-04T06:40:38,538 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a55795bb1224449fae1fa2b7b7bd55e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:38,538 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b33a1c823446a9b4ccce7505735dde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294436547 2024-12-04T06:40:38,538 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2690b41d255a460cb1c27d4ff68feeda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733294436162 2024-12-04T06:40:38,539 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc063ccbd9e54afe9a4a1fb88bf53430, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437242 2024-12-04T06:40:38,539 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6643705d1b4ee698f9cc65c65df5cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294436547 2024-12-04T06:40:38,541 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 148e839daa6a469e836a30b6ee01cced, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437884 2024-12-04T06:40:38,572 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:38,573 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/2c4ca276f532431d8d5822e2d97c7903 is 50, key is test_row_0/A:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:38,590 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:38,591 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/14240f60b2d141a4805886c81531b485 is 50, key is test_row_0/B:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:38,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741858_1034 (size=12241) 2024-12-04T06:40:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:38,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:40:38,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:38,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:38,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741859_1035 (size=12241) 2024-12-04T06:40:38,616 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/2c4ca276f532431d8d5822e2d97c7903 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/2c4ca276f532431d8d5822e2d97c7903 2024-12-04T06:40:38,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 is 50, key is test_row_0/A:col10/1733294437941/Put/seqid=0 2024-12-04T06:40:38,630 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/14240f60b2d141a4805886c81531b485 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/14240f60b2d141a4805886c81531b485 2024-12-04T06:40:38,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:38,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:38,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,719 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 2c4ca276f532431d8d5822e2d97c7903(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:38,719 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 14240f60b2d141a4805886c81531b485(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:38,719 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:38,719 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:38,720 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=12, startTime=1733294438532; duration=0sec 2024-12-04T06:40:38,720 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=12, startTime=1733294438532; duration=0sec 2024-12-04T06:40:38,720 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:38,720 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:38,720 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:38,722 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:38,722 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741860_1036 (size=14541) 2024-12-04T06:40:38,726 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:38,726 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:38,727 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,727 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/5243debfd76d45bdac14b20066af3cfa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=47.0 K 2024-12-04T06:40:38,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 2024-12-04T06:40:38,733 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5243debfd76d45bdac14b20066af3cfa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733294435945 2024-12-04T06:40:38,734 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 94116d2ff86443589a5ed6cf6a77b527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733294436162 2024-12-04T06:40:38,735 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a01ff9af9b084ccbb85b0d795f5ea588, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294436547 2024-12-04T06:40:38,736 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 85459510132f4047b388685bc5ec2ec0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437884 2024-12-04T06:40:38,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,746 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294498735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294498744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294498746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294498747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/964aa27388064c15a0810cd7e56290eb is 50, key is test_row_0/B:col10/1733294437941/Put/seqid=0 2024-12-04T06:40:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294498748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,772 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:38,773 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7740626659ac4e239c69cbf2ea715583 is 50, key is test_row_0/C:col10/1733294437895/Put/seqid=0 2024-12-04T06:40:38,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741861_1037 (size=12151) 2024-12-04T06:40:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:38,827 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T06:40:38,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741862_1038 (size=12241) 2024-12-04T06:40:38,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/964aa27388064c15a0810cd7e56290eb 2024-12-04T06:40:38,844 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7740626659ac4e239c69cbf2ea715583 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7740626659ac4e239c69cbf2ea715583 2024-12-04T06:40:38,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fba657e3d51a4346bcd158f9c5582535 is 50, key is test_row_0/C:col10/1733294437941/Put/seqid=0 2024-12-04T06:40:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294498857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294498858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294498858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294498858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,865 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 7740626659ac4e239c69cbf2ea715583(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:38,865 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:38,865 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=12, startTime=1733294438532; duration=0sec 2024-12-04T06:40:38,865 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:38,865 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:38,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:38,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:38,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294498867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741863_1039 (size=12151) 2024-12-04T06:40:39,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294499067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294499067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294499067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294499068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294499087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fba657e3d51a4346bcd158f9c5582535 2024-12-04T06:40:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 2024-12-04T06:40:39,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3, entries=200, sequenceid=135, filesize=14.2 K 2024-12-04T06:40:39,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/964aa27388064c15a0810cd7e56290eb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb 2024-12-04T06:40:39,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb, entries=150, sequenceid=135, filesize=11.9 K 2024-12-04T06:40:39,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fba657e3d51a4346bcd158f9c5582535 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535 2024-12-04T06:40:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:39,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535, entries=150, sequenceid=135, filesize=11.9 K 2024-12-04T06:40:39,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c8a5d67afecdb7afe2ddb08e0bd45184 in 754ms, sequenceid=135, compaction requested=false 2024-12-04T06:40:39,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:39,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:39,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:39,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc4f421f2a9449d191cbfb174ef3988d is 50, key is test_row_0/A:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:39,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294499400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294499402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294499406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294499408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294499408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741864_1040 (size=14541) 2024-12-04T06:40:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294499517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294499517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294499527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294499527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294499537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294499727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294499727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294499737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294499747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294499757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:39,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc4f421f2a9449d191cbfb174ef3988d 2024-12-04T06:40:39,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/010fdbb61d654376ad5cd2e0f5eb743e is 50, key is test_row_0/B:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:39,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741865_1041 (size=12151) 2024-12-04T06:40:39,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/010fdbb61d654376ad5cd2e0f5eb743e 2024-12-04T06:40:39,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a766c4109ef64882a30e76c04223d575 is 50, key is test_row_0/C:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:39,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:39,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:39,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:39,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741866_1042 (size=12151) 2024-12-04T06:40:39,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a766c4109ef64882a30e76c04223d575 2024-12-04T06:40:40,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc4f421f2a9449d191cbfb174ef3988d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d 2024-12-04T06:40:40,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d, entries=200, sequenceid=157, filesize=14.2 K 2024-12-04T06:40:40,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/010fdbb61d654376ad5cd2e0f5eb743e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e 2024-12-04T06:40:40,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e, entries=150, sequenceid=157, filesize=11.9 K 2024-12-04T06:40:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a766c4109ef64882a30e76c04223d575 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575 2024-12-04T06:40:40,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294500037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294500041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294500047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575, entries=150, sequenceid=157, filesize=11.9 K 2024-12-04T06:40:40,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c8a5d67afecdb7afe2ddb08e0bd45184 in 678ms, sequenceid=157, compaction requested=true 2024-12-04T06:40:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:40,058 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:40,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:40,058 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:40,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:40,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:40,061 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:40,061 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:40,061 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,061 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/14240f60b2d141a4805886c81531b485, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=35.7 K 2024-12-04T06:40:40,062 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 14240f60b2d141a4805886c81531b485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437884 2024-12-04T06:40:40,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:40,063 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 964aa27388064c15a0810cd7e56290eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733294437941 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:40,063 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,063 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:40,063 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,063 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/2c4ca276f532431d8d5822e2d97c7903, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=40.4 K 2024-12-04T06:40:40,064 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 010fdbb61d654376ad5cd2e0f5eb743e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:40,064 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4ca276f532431d8d5822e2d97c7903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437884 2024-12-04T06:40:40,065 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aa8bce2b26a4dc2beb4b7ec572a3cb3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733294437941 2024-12-04T06:40:40,066 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4f421f2a9449d191cbfb174ef3988d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:40,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc23441645624b82876a3bae3e64d1d9 is 50, key is test_row_0/A:col10/1733294439401/Put/seqid=0 2024-12-04T06:40:40,097 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:40,098 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:40,098 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8ad4386e7b134f55958a545cc032dec1 is 50, key is test_row_0/B:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:40,099 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5d1954339bcb4ceda72ecab6d6351d1d is 50, key is test_row_0/A:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741869_1045 (size=12493) 2024-12-04T06:40:40,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741868_1044 (size=12493) 2024-12-04T06:40:40,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741867_1043 (size=14541) 2024-12-04T06:40:40,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc23441645624b82876a3bae3e64d1d9 2024-12-04T06:40:40,138 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8ad4386e7b134f55958a545cc032dec1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8ad4386e7b134f55958a545cc032dec1 2024-12-04T06:40:40,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d4527d5c578f402a80ad88aa0415d3c5 is 50, key is test_row_0/B:col10/1733294439401/Put/seqid=0 2024-12-04T06:40:40,241 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 8ad4386e7b134f55958a545cc032dec1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:40,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:40,241 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294440058; duration=0sec 2024-12-04T06:40:40,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:40,242 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:40,242 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:40,244 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:40,244 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:40,244 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,244 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7740626659ac4e239c69cbf2ea715583, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=35.7 K 2024-12-04T06:40:40,245 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7740626659ac4e239c69cbf2ea715583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733294437884 2024-12-04T06:40:40,245 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fba657e3d51a4346bcd158f9c5582535, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733294437941 2024-12-04T06:40:40,246 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a766c4109ef64882a30e76c04223d575, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:40,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741870_1046 (size=12151) 2024-12-04T06:40:40,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d4527d5c578f402a80ad88aa0415d3c5 2024-12-04T06:40:40,263 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:40,264 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/3b7a140313134d2198a04737720f0daa is 50, key is test_row_0/C:col10/1733294438742/Put/seqid=0 2024-12-04T06:40:40,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294500267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294500267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/f98fe68f049e4f2eb2623884b1a54107 is 50, key is test_row_0/C:col10/1733294439401/Put/seqid=0 2024-12-04T06:40:40,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741871_1047 (size=12493) 2024-12-04T06:40:40,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741872_1048 (size=12151) 2024-12-04T06:40:40,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294500377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294500377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,523 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T06:40:40,523 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T06:40:40,525 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T06:40:40,525 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T06:40:40,526 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T06:40:40,526 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T06:40:40,527 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-04T06:40:40,527 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-04T06:40:40,545 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5d1954339bcb4ceda72ecab6d6351d1d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5d1954339bcb4ceda72ecab6d6351d1d 2024-12-04T06:40:40,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294500547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,561 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 5d1954339bcb4ceda72ecab6d6351d1d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:40,561 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:40,561 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294440058; duration=0sec 2024-12-04T06:40:40,561 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:40,562 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294500557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294500567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294500586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294500587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/f98fe68f049e4f2eb2623884b1a54107 2024-12-04T06:40:40,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/cc23441645624b82876a3bae3e64d1d9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9 2024-12-04T06:40:40,708 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/3b7a140313134d2198a04737720f0daa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/3b7a140313134d2198a04737720f0daa 2024-12-04T06:40:40,726 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 3b7a140313134d2198a04737720f0daa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:40,726 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:40,727 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294440058; duration=0sec 2024-12-04T06:40:40,727 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:40,727 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:40,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9, entries=200, sequenceid=171, filesize=14.2 K 2024-12-04T06:40:40,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d4527d5c578f402a80ad88aa0415d3c5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5 2024-12-04T06:40:40,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5, entries=150, sequenceid=171, filesize=11.9 K 2024-12-04T06:40:40,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/f98fe68f049e4f2eb2623884b1a54107 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107 2024-12-04T06:40:40,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107, entries=150, sequenceid=171, filesize=11.9 K 2024-12-04T06:40:40,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c8a5d67afecdb7afe2ddb08e0bd45184 in 732ms, sequenceid=171, compaction requested=false 2024-12-04T06:40:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:40,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:40,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:40,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/3f1ab0b3df5d41a09e3f0716834c709d is 50, key is test_row_0/A:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:40,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:40,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:40,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741873_1049 (size=14541) 2024-12-04T06:40:40,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294500965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294500968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:41,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294501077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294501087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:41,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294501286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294501296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/3f1ab0b3df5d41a09e3f0716834c709d 2024-12-04T06:40:41,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2cedc7a9495940d8ba78fc55f0e4bcb8 is 50, key is test_row_0/B:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741874_1050 (size=12151) 2024-12-04T06:40:41,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2cedc7a9495940d8ba78fc55f0e4bcb8 2024-12-04T06:40:41,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2a57a8b4657e4f76b18cbc4f1d800041 is 50, key is test_row_0/C:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741875_1051 (size=12151) 2024-12-04T06:40:41,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2a57a8b4657e4f76b18cbc4f1d800041 2024-12-04T06:40:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/3f1ab0b3df5d41a09e3f0716834c709d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d 2024-12-04T06:40:41,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d, entries=200, sequenceid=194, filesize=14.2 K 2024-12-04T06:40:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2cedc7a9495940d8ba78fc55f0e4bcb8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8 2024-12-04T06:40:41,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T06:40:41,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2a57a8b4657e4f76b18cbc4f1d800041 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041 2024-12-04T06:40:41,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T06:40:41,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c8a5d67afecdb7afe2ddb08e0bd45184 in 557ms, sequenceid=194, compaction requested=true 2024-12-04T06:40:41,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:41,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:41,456 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:41,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:41,456 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:41,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:41,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:41,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:41,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:41,458 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:41,458 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:41,459 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,459 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5d1954339bcb4ceda72ecab6d6351d1d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=40.6 K 2024-12-04T06:40:41,459 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:41,459 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:41,459 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,459 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8ad4386e7b134f55958a545cc032dec1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=35.9 K 2024-12-04T06:40:41,460 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1954339bcb4ceda72ecab6d6351d1d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:41,460 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad4386e7b134f55958a545cc032dec1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:41,461 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc23441645624b82876a3bae3e64d1d9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733294439401 2024-12-04T06:40:41,462 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d4527d5c578f402a80ad88aa0415d3c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733294439401 2024-12-04T06:40:41,462 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1ab0b3df5d41a09e3f0716834c709d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:41,462 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cedc7a9495940d8ba78fc55f0e4bcb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:41,479 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:41,479 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/0c9c9bf8e40a40309d58bcea9b3f6169 is 50, key is test_row_0/B:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:41,484 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:41,484 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/fb01dd09c8d44fe6b1c36aa786680dfa is 50, key is test_row_0/A:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:41,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741876_1052 (size=12595) 2024-12-04T06:40:41,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741877_1053 (size=12595) 2024-12-04T06:40:41,512 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/0c9c9bf8e40a40309d58bcea9b3f6169 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0c9c9bf8e40a40309d58bcea9b3f6169 2024-12-04T06:40:41,526 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 0c9c9bf8e40a40309d58bcea9b3f6169(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:41,526 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:41,526 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294441456; duration=0sec 2024-12-04T06:40:41,527 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:41,527 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:41,527 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:41,528 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:41,528 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:41,528 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,529 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/3b7a140313134d2198a04737720f0daa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=35.9 K 2024-12-04T06:40:41,529 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7a140313134d2198a04737720f0daa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733294438741 2024-12-04T06:40:41,531 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f98fe68f049e4f2eb2623884b1a54107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733294439401 2024-12-04T06:40:41,535 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a57a8b4657e4f76b18cbc4f1d800041, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:41,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-04T06:40:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:41,540 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7e39d8ba376a4d08aecb8492d3619bc5 is 50, key is test_row_0/A:col10/1733294440952/Put/seqid=0 2024-12-04T06:40:41,555 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:41,556 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/16d5e46facbc4c6995db4936a131bcee is 50, key is test_row_0/C:col10/1733294440120/Put/seqid=0 2024-12-04T06:40:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:41,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:41,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741878_1054 (size=12151) 2024-12-04T06:40:41,595 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7e39d8ba376a4d08aecb8492d3619bc5 2024-12-04T06:40:41,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741879_1055 (size=12595) 2024-12-04T06:40:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d101ce9c918f4748aee27361f196a174 is 50, key is test_row_0/B:col10/1733294440952/Put/seqid=0 2024-12-04T06:40:41,628 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/16d5e46facbc4c6995db4936a131bcee as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/16d5e46facbc4c6995db4936a131bcee 2024-12-04T06:40:41,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294501620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294501621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,638 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 16d5e46facbc4c6995db4936a131bcee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,638 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,638 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294441457; duration=0sec 2024-12-04T06:40:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294501621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,639 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294501624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,639 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741880_1056 (size=12151) 2024-12-04T06:40:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294501637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294501747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294501747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294501747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294501747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294501757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,912 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/fb01dd09c8d44fe6b1c36aa786680dfa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/fb01dd09c8d44fe6b1c36aa786680dfa 2024-12-04T06:40:41,921 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into fb01dd09c8d44fe6b1c36aa786680dfa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:41,921 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:41,921 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294441456; duration=0sec 2024-12-04T06:40:41,921 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:41,921 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294501957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294501957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294501957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294501967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294501967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,045 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d101ce9c918f4748aee27361f196a174 2024-12-04T06:40:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e6c2d2ca64784e3da1332281b556c081 is 50, key is test_row_0/C:col10/1733294440952/Put/seqid=0 2024-12-04T06:40:42,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741881_1057 (size=12151) 2024-12-04T06:40:42,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294502267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294502267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294502277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294502277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294502277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:42,467 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e6c2d2ca64784e3da1332281b556c081 2024-12-04T06:40:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7e39d8ba376a4d08aecb8492d3619bc5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5 2024-12-04T06:40:42,487 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T06:40:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d101ce9c918f4748aee27361f196a174 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174 2024-12-04T06:40:42,496 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T06:40:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e6c2d2ca64784e3da1332281b556c081 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081 2024-12-04T06:40:42,506 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T06:40:42,507 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c8a5d67afecdb7afe2ddb08e0bd45184 in 968ms, sequenceid=207, compaction requested=false 2024-12-04T06:40:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T06:40:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T06:40:42,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T06:40:42,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3110 sec 2024-12-04T06:40:42,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 4.3460 sec 2024-12-04T06:40:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:42,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:42,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:42,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5b0d9d3dd18460cbfc51781cb3bfd59 is 50, key is test_row_0/A:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741882_1058 (size=12151) 2024-12-04T06:40:42,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294502837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294502837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294502837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294502839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294502840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294502957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294502957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294502958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294502958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294502958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:42,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294503167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294503167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294503177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294503177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294503178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5b0d9d3dd18460cbfc51781cb3bfd59 2024-12-04T06:40:43,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/320bbafaba8543dbb721fbc50d7e1bc2 is 50, key is test_row_0/B:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:43,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741883_1059 (size=12151) 2024-12-04T06:40:43,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/320bbafaba8543dbb721fbc50d7e1bc2 2024-12-04T06:40:43,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/137bbf761c004ecba7ab42e121645dba is 50, key is test_row_0/C:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:43,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741884_1060 (size=12151) 2024-12-04T06:40:43,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294503486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294503487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294503487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294503487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294503487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:43,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/137bbf761c004ecba7ab42e121645dba 2024-12-04T06:40:43,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5b0d9d3dd18460cbfc51781cb3bfd59 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59 2024-12-04T06:40:43,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59, entries=150, sequenceid=230, filesize=11.9 K 2024-12-04T06:40:43,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/320bbafaba8543dbb721fbc50d7e1bc2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2 2024-12-04T06:40:43,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2, entries=150, sequenceid=230, filesize=11.9 K 2024-12-04T06:40:43,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/137bbf761c004ecba7ab42e121645dba as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba 2024-12-04T06:40:43,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba, entries=150, sequenceid=230, filesize=11.9 K 2024-12-04T06:40:43,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c8a5d67afecdb7afe2ddb08e0bd45184 in 906ms, sequenceid=230, compaction requested=true 2024-12-04T06:40:43,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:43,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:43,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:43,698 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:43,698 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:43,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:43,703 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:43,703 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:43,703 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:43,703 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0c9c9bf8e40a40309d58bcea9b3f6169, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.0 K 2024-12-04T06:40:43,703 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:43,704 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:43,704 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:43,704 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9c9bf8e40a40309d58bcea9b3f6169, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:43,704 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/fb01dd09c8d44fe6b1c36aa786680dfa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.0 K 2024-12-04T06:40:43,704 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d101ce9c918f4748aee27361f196a174, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294440952 2024-12-04T06:40:43,704 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb01dd09c8d44fe6b1c36aa786680dfa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:43,705 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 320bbafaba8543dbb721fbc50d7e1bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:43,705 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e39d8ba376a4d08aecb8492d3619bc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294440952 2024-12-04T06:40:43,708 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5b0d9d3dd18460cbfc51781cb3bfd59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:43,724 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:43,725 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/b8fb083f043c499591218408e7b0c7e6 is 50, key is test_row_0/B:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:43,727 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:43,729 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/a18b493299484087ad0a967ca91faca4 is 50, key is test_row_0/A:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:43,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741886_1062 (size=12697) 2024-12-04T06:40:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741885_1061 (size=12697) 2024-12-04T06:40:43,766 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/a18b493299484087ad0a967ca91faca4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a18b493299484087ad0a967ca91faca4 2024-12-04T06:40:43,777 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into a18b493299484087ad0a967ca91faca4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:43,777 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:43,778 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294443698; duration=0sec 2024-12-04T06:40:43,778 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:43,778 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:43,778 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:43,781 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:43,781 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:43,782 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:43,782 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/16d5e46facbc4c6995db4936a131bcee, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.0 K 2024-12-04T06:40:43,782 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d5e46facbc4c6995db4936a131bcee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294440120 2024-12-04T06:40:43,783 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c2d2ca64784e3da1332281b556c081, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294440952 2024-12-04T06:40:43,783 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137bbf761c004ecba7ab42e121645dba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:43,799 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#50 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:43,800 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/27710b3d986b4ce6a9ee7b832877a29e is 50, key is test_row_0/C:col10/1733294441634/Put/seqid=0 2024-12-04T06:40:43,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741887_1063 (size=12697) 2024-12-04T06:40:43,840 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/27710b3d986b4ce6a9ee7b832877a29e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/27710b3d986b4ce6a9ee7b832877a29e 2024-12-04T06:40:43,859 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 27710b3d986b4ce6a9ee7b832877a29e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:43,859 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:43,860 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294443699; duration=0sec 2024-12-04T06:40:43,860 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:43,861 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:43,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:43,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:44,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/caace600ca13479094643800ac0fd7cc is 50, key is test_row_0/A:col10/1733294442836/Put/seqid=0 2024-12-04T06:40:44,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741888_1064 (size=14541) 2024-12-04T06:40:44,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/caace600ca13479094643800ac0fd7cc 2024-12-04T06:40:44,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7f81394b84e64348b2c2c90438f578ac is 50, key is test_row_0/B:col10/1733294442836/Put/seqid=0 2024-12-04T06:40:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294504023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294504025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294504026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294504028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294504029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741889_1065 (size=12151) 2024-12-04T06:40:44,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7f81394b84e64348b2c2c90438f578ac 2024-12-04T06:40:44,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7760378fd14e4d58a9693161e77079c4 is 50, key is test_row_0/C:col10/1733294442836/Put/seqid=0 2024-12-04T06:40:44,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741890_1066 (size=12151) 2024-12-04T06:40:44,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294504137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294504141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294504141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294504144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294504144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,164 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/b8fb083f043c499591218408e7b0c7e6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/b8fb083f043c499591218408e7b0c7e6 2024-12-04T06:40:44,177 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into b8fb083f043c499591218408e7b0c7e6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:44,177 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:44,177 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294443698; duration=0sec 2024-12-04T06:40:44,177 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:44,177 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:44,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294504347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294504357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294504357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294504357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294504358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7760378fd14e4d58a9693161e77079c4 2024-12-04T06:40:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/caace600ca13479094643800ac0fd7cc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc 2024-12-04T06:40:44,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc, entries=200, sequenceid=244, filesize=14.2 K 2024-12-04T06:40:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7f81394b84e64348b2c2c90438f578ac as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac 2024-12-04T06:40:44,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac, entries=150, sequenceid=244, filesize=11.9 K 2024-12-04T06:40:44,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7760378fd14e4d58a9693161e77079c4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4 2024-12-04T06:40:44,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4, entries=150, sequenceid=244, filesize=11.9 K 2024-12-04T06:40:44,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c8a5d67afecdb7afe2ddb08e0bd45184 in 503ms, sequenceid=244, compaction requested=false 2024-12-04T06:40:44,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:44,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:44,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:44,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/551ceb208ea640c693103312f78f89a7 is 50, key is test_row_0/A:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:44,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741891_1067 (size=14691) 2024-12-04T06:40:44,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/551ceb208ea640c693103312f78f89a7 2024-12-04T06:40:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294504687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294504687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294504690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/88bb7f61134f47fcb121d6568b1c0328 is 50, key is test_row_0/B:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294504690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294504692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741892_1068 (size=12301) 2024-12-04T06:40:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294504797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294504797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294504797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294504806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294504806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294505006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294505006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294505007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294505017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294505017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/88bb7f61134f47fcb121d6568b1c0328 2024-12-04T06:40:45,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/1b53c006da1243a9b344dee910f154cf is 50, key is test_row_0/C:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:45,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741893_1069 (size=12301) 2024-12-04T06:40:45,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294505317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294505317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294505317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294505326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294505337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/1b53c006da1243a9b344dee910f154cf 2024-12-04T06:40:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/551ceb208ea640c693103312f78f89a7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7 2024-12-04T06:40:45,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7, entries=200, sequenceid=266, filesize=14.3 K 2024-12-04T06:40:45,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/88bb7f61134f47fcb121d6568b1c0328 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328 2024-12-04T06:40:45,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328, entries=150, sequenceid=266, filesize=12.0 K 2024-12-04T06:40:45,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/1b53c006da1243a9b344dee910f154cf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf 2024-12-04T06:40:45,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf, entries=150, sequenceid=266, filesize=12.0 K 2024-12-04T06:40:45,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c8a5d67afecdb7afe2ddb08e0bd45184 in 903ms, sequenceid=266, compaction requested=true 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:45,565 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:45,565 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:45,567 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41929 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:45,567 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:45,567 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:45,567 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a18b493299484087ad0a967ca91faca4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=40.9 K 2024-12-04T06:40:45,567 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:45,567 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:45,567 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:45,568 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/b8fb083f043c499591218408e7b0c7e6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.3 K 2024-12-04T06:40:45,568 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18b493299484087ad0a967ca91faca4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:45,568 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b8fb083f043c499591218408e7b0c7e6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:45,569 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting caace600ca13479094643800ac0fd7cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294442796 2024-12-04T06:40:45,569 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f81394b84e64348b2c2c90438f578ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294442816 2024-12-04T06:40:45,569 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bb7f61134f47fcb121d6568b1c0328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444024 2024-12-04T06:40:45,570 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 551ceb208ea640c693103312f78f89a7, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444003 2024-12-04T06:40:45,588 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:45,589 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ca87f877b6dd442e880c78fd331e7560 is 50, key is test_row_0/B:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:45,592 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:45,593 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/b4dad0b7eae74378813c9446ac111a3f is 50, key is test_row_0/A:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741894_1070 (size=12949) 2024-12-04T06:40:45,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741895_1071 (size=12949) 2024-12-04T06:40:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:45,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:45,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7538316e60cd4614a237989fde5e9f19 is 50, key is test_row_0/A:col10/1733294445828/Put/seqid=0 2024-12-04T06:40:45,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741896_1072 (size=14741) 2024-12-04T06:40:45,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294505888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294505892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294505892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294505892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294505894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:45,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294505997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,012 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ca87f877b6dd442e880c78fd331e7560 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ca87f877b6dd442e880c78fd331e7560 2024-12-04T06:40:46,012 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/b4dad0b7eae74378813c9446ac111a3f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/b4dad0b7eae74378813c9446ac111a3f 2024-12-04T06:40:46,021 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into ca87f877b6dd442e880c78fd331e7560(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:46,022 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:46,022 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into b4dad0b7eae74378813c9446ac111a3f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:46,022 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294445565; duration=0sec 2024-12-04T06:40:46,022 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:46,022 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294445565; duration=0sec 2024-12-04T06:40:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,022 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:46,022 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294506017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,022 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294506017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294506017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294506017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,023 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:46,023 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:46,025 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:46,025 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:46,025 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:46,025 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/27710b3d986b4ce6a9ee7b832877a29e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.3 K 2024-12-04T06:40:46,025 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 27710b3d986b4ce6a9ee7b832877a29e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733294441621 2024-12-04T06:40:46,026 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7760378fd14e4d58a9693161e77079c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294442816 2024-12-04T06:40:46,026 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b53c006da1243a9b344dee910f154cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444024 2024-12-04T06:40:46,035 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:46,035 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/86d3cea0df1e4e14beef1754ee49a1e9 is 50, key is test_row_0/C:col10/1733294444028/Put/seqid=0 2024-12-04T06:40:46,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741897_1073 (size=12949) 2024-12-04T06:40:46,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294506207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294506227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294506227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294506227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294506227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7538316e60cd4614a237989fde5e9f19 2024-12-04T06:40:46,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/74964120ffec42cf8644381a6a5100c2 is 50, key is test_row_0/B:col10/1733294445828/Put/seqid=0 2024-12-04T06:40:46,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741898_1074 (size=12301) 2024-12-04T06:40:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T06:40:46,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:46,398 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-04T06:40:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T06:40:46,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:46,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:46,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:46,457 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/86d3cea0df1e4e14beef1754ee49a1e9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/86d3cea0df1e4e14beef1754ee49a1e9 2024-12-04T06:40:46,464 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 86d3cea0df1e4e14beef1754ee49a1e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:46,464 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:46,464 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294445565; duration=0sec 2024-12-04T06:40:46,464 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:46,465 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T06:40:46,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294506517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294506536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294506536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294506536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294506537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:46,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T06:40:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:46,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:46,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/74964120ffec42cf8644381a6a5100c2 2024-12-04T06:40:46,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8355c788426a4dada0bdbe714bf0494b is 50, key is test_row_0/C:col10/1733294445828/Put/seqid=0 2024-12-04T06:40:46,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741899_1075 (size=12301) 2024-12-04T06:40:46,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8355c788426a4dada0bdbe714bf0494b 2024-12-04T06:40:46,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/7538316e60cd4614a237989fde5e9f19 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19 2024-12-04T06:40:46,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19, entries=200, sequenceid=280, filesize=14.4 K 2024-12-04T06:40:46,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/74964120ffec42cf8644381a6a5100c2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2 2024-12-04T06:40:46,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2, entries=150, sequenceid=280, filesize=12.0 K 2024-12-04T06:40:46,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8355c788426a4dada0bdbe714bf0494b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b 2024-12-04T06:40:46,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b, entries=150, sequenceid=280, filesize=12.0 K 2024-12-04T06:40:46,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c8a5d67afecdb7afe2ddb08e0bd45184 in 876ms, sequenceid=280, compaction requested=false 2024-12-04T06:40:46,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:46,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-04T06:40:46,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:46,708 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ed0b0eead63c4df7ae38e0b02691c155 is 50, key is test_row_0/A:col10/1733294445891/Put/seqid=0 2024-12-04T06:40:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T06:40:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741900_1076 (size=12301) 2024-12-04T06:40:46,725 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ed0b0eead63c4df7ae38e0b02691c155 2024-12-04T06:40:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/bb7e05cacd464c4c899dd4e4f5804b7b is 50, key is test_row_0/B:col10/1733294445891/Put/seqid=0 2024-12-04T06:40:46,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741901_1077 (size=12301) 2024-12-04T06:40:46,742 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/bb7e05cacd464c4c899dd4e4f5804b7b 2024-12-04T06:40:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7d468837de25424fa53c8f49c2446840 is 50, key is test_row_0/C:col10/1733294445891/Put/seqid=0 2024-12-04T06:40:46,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741902_1078 (size=12301) 2024-12-04T06:40:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T06:40:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294507057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294507057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294507058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294507059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294507064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,167 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7d468837de25424fa53c8f49c2446840 2024-12-04T06:40:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294507167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/ed0b0eead63c4df7ae38e0b02691c155 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155 2024-12-04T06:40:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294507177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294507177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:40:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/bb7e05cacd464c4c899dd4e4f5804b7b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b 2024-12-04T06:40:47,190 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:40:47,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/7d468837de25424fa53c8f49c2446840 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840 2024-12-04T06:40:47,199 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:40:47,200 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c8a5d67afecdb7afe2ddb08e0bd45184 in 491ms, sequenceid=301, compaction requested=true 2024-12-04T06:40:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T06:40:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T06:40:47,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T06:40:47,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 801 msec 2024-12-04T06:40:47,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 807 msec 2024-12-04T06:40:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:47,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:47,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:47,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e42922202cb9421cb5775896f3bb0d72 is 50, key is test_row_0/A:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:47,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741903_1079 (size=14741) 2024-12-04T06:40:47,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e42922202cb9421cb5775896f3bb0d72 2024-12-04T06:40:47,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6788d0820284466d9898ade9ba4151a0 is 50, key is test_row_0/B:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:47,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741904_1080 (size=12301) 2024-12-04T06:40:47,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6788d0820284466d9898ade9ba4151a0 2024-12-04T06:40:47,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/659131fc5dbe41e18aa3ddffee91da07 is 50, key is test_row_0/C:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294507411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741905_1081 (size=12301) 2024-12-04T06:40:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294507421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294507422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294507527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T06:40:47,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:47,537 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-04T06:40:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:47,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:47,541 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:47,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294507536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294507536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:47,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T06:40:47,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294507737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294507746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294507746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:47,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/659131fc5dbe41e18aa3ddffee91da07 2024-12-04T06:40:47,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e42922202cb9421cb5775896f3bb0d72 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72 2024-12-04T06:40:47,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72, entries=200, sequenceid=316, filesize=14.4 K 2024-12-04T06:40:47,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6788d0820284466d9898ade9ba4151a0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0 2024-12-04T06:40:47,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0, entries=150, sequenceid=316, filesize=12.0 K 2024-12-04T06:40:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/659131fc5dbe41e18aa3ddffee91da07 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07 2024-12-04T06:40:47,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T06:40:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07, entries=150, sequenceid=316, filesize=12.0 K 2024-12-04T06:40:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:47,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c8a5d67afecdb7afe2ddb08e0bd45184 in 473ms, sequenceid=316, compaction requested=true 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:47,851 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:47,851 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:47,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:47,853 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:47,853 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:47,853 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,853 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/b4dad0b7eae74378813c9446ac111a3f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=53.4 K 2024-12-04T06:40:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:47,861 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4dad0b7eae74378813c9446ac111a3f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444024 2024-12-04T06:40:47,861 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7538316e60cd4614a237989fde5e9f19, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733294444686 2024-12-04T06:40:47,861 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:47,861 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:47,861 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:47,862 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0b0eead63c4df7ae38e0b02691c155, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294445888 2024-12-04T06:40:47,862 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ca87f877b6dd442e880c78fd331e7560, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=48.7 K 2024-12-04T06:40:47,862 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42922202cb9421cb5775896f3bb0d72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:47,863 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ca87f877b6dd442e880c78fd331e7560, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444024 2024-12-04T06:40:47,863 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 74964120ffec42cf8644381a6a5100c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733294445827 2024-12-04T06:40:47,864 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7e05cacd464c4c899dd4e4f5804b7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294445888 2024-12-04T06:40:47,864 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6788d0820284466d9898ade9ba4151a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:47,876 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:47,878 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/3d57c792b81c4949b6cf548566609961 is 50, key is test_row_0/A:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:47,880 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:47,880 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/498719cf8d4e4fd0a5edc41b4be74b9c is 50, key is test_row_0/B:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:47,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741906_1082 (size=13085) 2024-12-04T06:40:47,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741907_1083 (size=13085) 2024-12-04T06:40:48,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-04T06:40:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:48,003 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:48,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/0ce4be02f6774ee58db5c86f24e7b6fa is 50, key is test_row_0/A:col10/1733294447407/Put/seqid=0 2024-12-04T06:40:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741908_1084 (size=12301) 2024-12-04T06:40:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:48,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294508077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294508077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294508078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294508078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294508080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,086 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:48,084 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:48,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294508186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294508187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294508187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,307 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/3d57c792b81c4949b6cf548566609961 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3d57c792b81c4949b6cf548566609961 2024-12-04T06:40:48,313 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/498719cf8d4e4fd0a5edc41b4be74b9c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/498719cf8d4e4fd0a5edc41b4be74b9c 2024-12-04T06:40:48,323 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 498719cf8d4e4fd0a5edc41b4be74b9c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:48,324 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:48,324 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=12, startTime=1733294447851; duration=0sec 2024-12-04T06:40:48,324 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:48,324 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:48,324 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:48,325 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 3d57c792b81c4949b6cf548566609961(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:48,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:48,325 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=12, startTime=1733294447851; duration=0sec 2024-12-04T06:40:48,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:48,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:48,326 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:48,326 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:48,326 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:48,326 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/86d3cea0df1e4e14beef1754ee49a1e9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=48.7 K 2024-12-04T06:40:48,327 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d3cea0df1e4e14beef1754ee49a1e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733294444024 2024-12-04T06:40:48,327 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8355c788426a4dada0bdbe714bf0494b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733294445827 2024-12-04T06:40:48,328 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d468837de25424fa53c8f49c2446840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294445888 2024-12-04T06:40:48,328 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 659131fc5dbe41e18aa3ddffee91da07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:48,343 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:48,344 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/1731cbb322bd41fd9538a4fb87a52f7e is 50, key is test_row_0/C:col10/1733294447377/Put/seqid=0 2024-12-04T06:40:48,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741909_1085 (size=13085) 2024-12-04T06:40:48,359 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/1731cbb322bd41fd9538a4fb87a52f7e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1731cbb322bd41fd9538a4fb87a52f7e 2024-12-04T06:40:48,366 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 1731cbb322bd41fd9538a4fb87a52f7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:48,366 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:48,366 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=12, startTime=1733294447851; duration=0sec 2024-12-04T06:40:48,366 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:48,366 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:48,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294508397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294508397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294508397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,423 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/0ce4be02f6774ee58db5c86f24e7b6fa 2024-12-04T06:40:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/781b48d1892c40c5af20160b18fce753 is 50, key is test_row_0/B:col10/1733294447407/Put/seqid=0 2024-12-04T06:40:48,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741910_1086 (size=12301) 2024-12-04T06:40:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294508706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294508706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294508706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:48,846 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/781b48d1892c40c5af20160b18fce753 2024-12-04T06:40:48,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/0ae0b43c48d14ebf8da637ca71453b3c is 50, key is test_row_0/C:col10/1733294447407/Put/seqid=0 2024-12-04T06:40:48,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741911_1087 (size=12301) 2024-12-04T06:40:48,861 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/0ae0b43c48d14ebf8da637ca71453b3c 2024-12-04T06:40:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/0ce4be02f6774ee58db5c86f24e7b6fa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa 2024-12-04T06:40:48,872 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa, entries=150, sequenceid=334, filesize=12.0 K 2024-12-04T06:40:48,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/781b48d1892c40c5af20160b18fce753 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753 2024-12-04T06:40:48,879 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753, entries=150, sequenceid=334, filesize=12.0 K 2024-12-04T06:40:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/0ae0b43c48d14ebf8da637ca71453b3c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c 2024-12-04T06:40:48,886 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c, entries=150, sequenceid=334, filesize=12.0 K 2024-12-04T06:40:48,887 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c8a5d67afecdb7afe2ddb08e0bd45184 in 884ms, sequenceid=334, compaction requested=false 2024-12-04T06:40:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T06:40:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T06:40:48,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T06:40:48,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-12-04T06:40:48,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3570 sec 2024-12-04T06:40:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:49,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/c6977ed664dd4c7aaca1cdefee592bf8 is 50, key is test_row_0/A:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741912_1088 (size=14741) 2024-12-04T06:40:49,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294509246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294509246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294509247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294509356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294509356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294509356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294509566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294509566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294509577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/c6977ed664dd4c7aaca1cdefee592bf8 2024-12-04T06:40:49,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2247cc126fc54c918524b274872844eb is 50, key is test_row_0/B:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741913_1089 (size=12301) 2024-12-04T06:40:49,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2247cc126fc54c918524b274872844eb 2024-12-04T06:40:49,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ae89e573b645450487bef80eba86416c is 50, key is test_row_0/C:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T06:40:49,687 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:49,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741914_1090 (size=12301) 2024-12-04T06:40:49,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ae89e573b645450487bef80eba86416c 2024-12-04T06:40:49,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-04T06:40:49,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:49,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:49,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T06:40:49,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/c6977ed664dd4c7aaca1cdefee592bf8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8 2024-12-04T06:40:49,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8, entries=200, sequenceid=352, filesize=14.4 K 2024-12-04T06:40:49,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/2247cc126fc54c918524b274872844eb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb 2024-12-04T06:40:49,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T06:40:49,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ae89e573b645450487bef80eba86416c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c 2024-12-04T06:40:49,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c, entries=150, sequenceid=352, filesize=12.0 K 2024-12-04T06:40:49,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c8a5d67afecdb7afe2ddb08e0bd45184 in 507ms, sequenceid=352, compaction requested=true 2024-12-04T06:40:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:49,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:49,727 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:49,727 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:49,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:49,728 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:49,728 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:49,728 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:49,728 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:49,728 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/498719cf8d4e4fd0a5edc41b4be74b9c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.8 K 2024-12-04T06:40:49,728 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:49,728 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:49,728 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 498719cf8d4e4fd0a5edc41b4be74b9c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:49,728 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3d57c792b81c4949b6cf548566609961, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=39.2 K 2024-12-04T06:40:49,729 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 781b48d1892c40c5af20160b18fce753, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733294447407 2024-12-04T06:40:49,729 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d57c792b81c4949b6cf548566609961, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:49,729 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2247cc126fc54c918524b274872844eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:49,730 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce4be02f6774ee58db5c86f24e7b6fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733294447407 2024-12-04T06:40:49,730 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6977ed664dd4c7aaca1cdefee592bf8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:49,747 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:49,747 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/cde165af65bc4333b9c5d027d76f0fda is 50, key is test_row_0/B:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,754 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:49,755 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/77a65b8f98f94861b16e1ae72ac51e19 is 50, key is test_row_0/A:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741915_1091 (size=13187) 2024-12-04T06:40:49,783 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/cde165af65bc4333b9c5d027d76f0fda as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/cde165af65bc4333b9c5d027d76f0fda 2024-12-04T06:40:49,793 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into cde165af65bc4333b9c5d027d76f0fda(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:49,793 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:49,793 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294449727; duration=0sec 2024-12-04T06:40:49,793 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:49,793 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:49,793 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:49,795 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:49,796 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:49,796 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:49,796 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1731cbb322bd41fd9538a4fb87a52f7e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.8 K 2024-12-04T06:40:49,797 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1731cbb322bd41fd9538a4fb87a52f7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733294447037 2024-12-04T06:40:49,797 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae0b43c48d14ebf8da637ca71453b3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733294447407 2024-12-04T06:40:49,798 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ae89e573b645450487bef80eba86416c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:49,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741916_1092 (size=13187) 2024-12-04T06:40:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T06:40:49,813 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#80 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:49,817 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/77a65b8f98f94861b16e1ae72ac51e19 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/77a65b8f98f94861b16e1ae72ac51e19 2024-12-04T06:40:49,817 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/330575ebd1e74b4fbea75ac3158eacaf is 50, key is test_row_0/C:col10/1733294448078/Put/seqid=0 2024-12-04T06:40:49,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-04T06:40:49,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:49,854 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:40:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:49,855 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 77a65b8f98f94861b16e1ae72ac51e19(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,855 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:49,855 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294449726; duration=0sec 2024-12-04T06:40:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:49,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:49,856 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:49,856 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741917_1093 (size=13187) 2024-12-04T06:40:49,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/63c906ebafe0466798d5675266fa5f1f is 50, key is test_row_0/A:col10/1733294449241/Put/seqid=0 2024-12-04T06:40:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:49,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:49,885 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/330575ebd1e74b4fbea75ac3158eacaf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/330575ebd1e74b4fbea75ac3158eacaf 2024-12-04T06:40:49,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741918_1094 (size=12301) 2024-12-04T06:40:49,896 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 330575ebd1e74b4fbea75ac3158eacaf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:49,896 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:49,896 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294449727; duration=0sec 2024-12-04T06:40:49,896 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:49,896 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:49,898 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/63c906ebafe0466798d5675266fa5f1f 2024-12-04T06:40:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/fe1d6b224a574458ab791c0b075f6cda is 50, key is test_row_0/B:col10/1733294449241/Put/seqid=0 2024-12-04T06:40:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294509911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294509917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294509918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741919_1095 (size=12301) 2024-12-04T06:40:49,932 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/fe1d6b224a574458ab791c0b075f6cda 2024-12-04T06:40:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ec8fb3c1c76841468426bafddc538be2 is 50, key is test_row_0/C:col10/1733294449241/Put/seqid=0 2024-12-04T06:40:49,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741920_1096 (size=12301) 2024-12-04T06:40:49,955 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ec8fb3c1c76841468426bafddc538be2 2024-12-04T06:40:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/63c906ebafe0466798d5675266fa5f1f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f 2024-12-04T06:40:49,974 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f, entries=150, sequenceid=371, filesize=12.0 K 2024-12-04T06:40:49,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/fe1d6b224a574458ab791c0b075f6cda as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda 2024-12-04T06:40:49,987 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda, entries=150, sequenceid=371, filesize=12.0 K 2024-12-04T06:40:49,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/ec8fb3c1c76841468426bafddc538be2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2 2024-12-04T06:40:49,998 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2, entries=150, sequenceid=371, filesize=12.0 K 2024-12-04T06:40:49,999 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c8a5d67afecdb7afe2ddb08e0bd45184 in 145ms, sequenceid=371, compaction requested=false 2024-12-04T06:40:49,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:49,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:49,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T06:40:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T06:40:50,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T06:40:50,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 306 msec 2024-12-04T06:40:50,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 313 msec 2024-12-04T06:40:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T06:40:50,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:50,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-04T06:40:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T06:40:50,021 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:50,022 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:50,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:50,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:50,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/984690e298774589bfe944cd45d7c03d is 50, key is test_row_0/A:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294510054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294510056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294510058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741921_1097 (size=14741) 2024-12-04T06:40:50,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/984690e298774589bfe944cd45d7c03d 2024-12-04T06:40:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294510098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,100 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294510106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/04ea5a14ae8a43b6b5d08bb80e394657 is 50, key is test_row_0/B:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,112 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T06:40:50,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741922_1098 (size=12301) 2024-12-04T06:40:50,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/04ea5a14ae8a43b6b5d08bb80e394657 2024-12-04T06:40:50,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294510167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294510167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T06:40:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294510177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/49593a026f34460e85683d92c996b6bb is 50, key is test_row_0/C:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741923_1099 (size=12301) 2024-12-04T06:40:50,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/49593a026f34460e85683d92c996b6bb 2024-12-04T06:40:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/984690e298774589bfe944cd45d7c03d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d 2024-12-04T06:40:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d, entries=200, sequenceid=388, filesize=14.4 K 2024-12-04T06:40:50,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/04ea5a14ae8a43b6b5d08bb80e394657 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657 2024-12-04T06:40:50,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657, entries=150, sequenceid=388, filesize=12.0 K 2024-12-04T06:40:50,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/49593a026f34460e85683d92c996b6bb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb 2024-12-04T06:40:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb, entries=150, sequenceid=388, filesize=12.0 K 2024-12-04T06:40:50,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c8a5d67afecdb7afe2ddb08e0bd45184 in 227ms, sequenceid=388, compaction requested=true 2024-12-04T06:40:50,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:50,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:50,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:50,256 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:50,256 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:50,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:50,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:50,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:50,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:50,258 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:50,258 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:50,258 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,258 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/77a65b8f98f94861b16e1ae72ac51e19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=39.3 K 2024-12-04T06:40:50,260 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:50,260 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:50,260 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,261 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/cde165af65bc4333b9c5d027d76f0fda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.9 K 2024-12-04T06:40:50,262 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a65b8f98f94861b16e1ae72ac51e19, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:50,263 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cde165af65bc4333b9c5d027d76f0fda, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:50,263 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1d6b224a574458ab791c0b075f6cda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733294449239 2024-12-04T06:40:50,263 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c906ebafe0466798d5675266fa5f1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733294449239 2024-12-04T06:40:50,264 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ea5a14ae8a43b6b5d08bb80e394657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449915 2024-12-04T06:40:50,264 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984690e298774589bfe944cd45d7c03d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449908 2024-12-04T06:40:50,276 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:50,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8d2d07d2e07a4c449d3ac24ab207251d is 50, key is test_row_0/B:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,282 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:50,282 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/040ce368a2fe41828382fdfbad2478d4 is 50, key is test_row_0/A:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741925_1101 (size=13289) 2024-12-04T06:40:50,312 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/040ce368a2fe41828382fdfbad2478d4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/040ce368a2fe41828382fdfbad2478d4 2024-12-04T06:40:50,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741924_1100 (size=13289) 2024-12-04T06:40:50,320 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 040ce368a2fe41828382fdfbad2478d4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:50,321 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:50,321 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294450256; duration=0sec 2024-12-04T06:40:50,321 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:50,321 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:50,321 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:50,322 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:50,322 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:50,322 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,322 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/330575ebd1e74b4fbea75ac3158eacaf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=36.9 K 2024-12-04T06:40:50,323 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8d2d07d2e07a4c449d3ac24ab207251d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8d2d07d2e07a4c449d3ac24ab207251d 2024-12-04T06:40:50,323 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330575ebd1e74b4fbea75ac3158eacaf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733294448073 2024-12-04T06:40:50,324 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec8fb3c1c76841468426bafddc538be2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1733294449239 2024-12-04T06:40:50,326 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49593a026f34460e85683d92c996b6bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449915 2024-12-04T06:40:50,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-04T06:40:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,331 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T06:40:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:50,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T06:40:50,342 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 8d2d07d2e07a4c449d3ac24ab207251d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:50,342 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:50,342 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294450256; duration=0sec 2024-12-04T06:40:50,342 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:50,342 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:50,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5099e56518541e0a04628298270413a is 50, key is test_row_0/A:col10/1733294450033/Put/seqid=0 2024-12-04T06:40:50,363 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:50,363 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/0dfbd1df5e1348ed95dc4853d30dd655 is 50, key is test_row_0/C:col10/1733294449916/Put/seqid=0 2024-12-04T06:40:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:50,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741927_1103 (size=13289) 2024-12-04T06:40:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741926_1102 (size=12301) 2024-12-04T06:40:50,425 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5099e56518541e0a04628298270413a 2024-12-04T06:40:50,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8fb3ce356029436485e674a3bffa8268 is 50, key is test_row_0/B:col10/1733294450033/Put/seqid=0 2024-12-04T06:40:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294510451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294510453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294510466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741928_1104 (size=12301) 2024-12-04T06:40:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294510577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294510577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294510577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T06:40:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294510787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294510796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294510797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:50,837 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/0dfbd1df5e1348ed95dc4853d30dd655 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0dfbd1df5e1348ed95dc4853d30dd655 2024-12-04T06:40:50,849 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 0dfbd1df5e1348ed95dc4853d30dd655(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:50,849 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:50,849 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294450257; duration=0sec 2024-12-04T06:40:50,849 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:50,849 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:50,888 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8fb3ce356029436485e674a3bffa8268 2024-12-04T06:40:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/cf9bffdda8d0491d8e4758431fcf7650 is 50, key is test_row_0/C:col10/1733294450033/Put/seqid=0 2024-12-04T06:40:50,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741929_1105 (size=12301) 2024-12-04T06:40:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/cf9bffdda8d0491d8e4758431fcf7650 2024-12-04T06:40:50,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f5099e56518541e0a04628298270413a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a 2024-12-04T06:40:50,949 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a, entries=150, sequenceid=405, filesize=12.0 K 2024-12-04T06:40:50,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/8fb3ce356029436485e674a3bffa8268 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268 2024-12-04T06:40:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268, entries=150, sequenceid=405, filesize=12.0 K 2024-12-04T06:40:50,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/cf9bffdda8d0491d8e4758431fcf7650 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650 2024-12-04T06:40:50,968 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650, entries=150, sequenceid=405, filesize=12.0 K 2024-12-04T06:40:50,970 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c8a5d67afecdb7afe2ddb08e0bd45184 in 638ms, sequenceid=405, compaction requested=false 2024-12-04T06:40:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:50,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T06:40:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T06:40:50,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T06:40:50,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 949 msec 2024-12-04T06:40:50,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 957 msec 2024-12-04T06:40:51,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:51,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5ac45369e1b644459cdfde4b191106fa is 50, key is test_row_0/A:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294511148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294511151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294511154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T06:40:51,157 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:51,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-04T06:40:51,162 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:51,164 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:51,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741930_1106 (size=14741) 2024-12-04T06:40:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:51,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294511256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294511267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294511267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:51,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:51,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294511477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294511480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294511478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5ac45369e1b644459cdfde4b191106fa 2024-12-04T06:40:51,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d638eb0d75b748738f2ca7fa1a2e99dc is 50, key is test_row_0/B:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:51,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741931_1107 (size=12301) 2024-12-04T06:40:51,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:51,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294511796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294511797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294511797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:51,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:51,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:52,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d638eb0d75b748738f2ca7fa1a2e99dc 2024-12-04T06:40:52,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d7d969d9c90e4e8e9d8219ed5e2561d3 is 50, key is test_row_0/C:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:52,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741932_1108 (size=12301) 2024-12-04T06:40:52,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d7d969d9c90e4e8e9d8219ed5e2561d3 2024-12-04T06:40:52,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5ac45369e1b644459cdfde4b191106fa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa 2024-12-04T06:40:52,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa, entries=200, sequenceid=424, filesize=14.4 K 2024-12-04T06:40:52,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d638eb0d75b748738f2ca7fa1a2e99dc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc 2024-12-04T06:40:52,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:52,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc, entries=150, sequenceid=424, filesize=12.0 K 2024-12-04T06:40:52,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d7d969d9c90e4e8e9d8219ed5e2561d3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3 2024-12-04T06:40:52,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3, entries=150, sequenceid=424, filesize=12.0 K 2024-12-04T06:40:52,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c8a5d67afecdb7afe2ddb08e0bd45184 in 1015ms, sequenceid=424, compaction requested=true 2024-12-04T06:40:52,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:52,117 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:52,119 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:52,119 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:52,119 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,119 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/040ce368a2fe41828382fdfbad2478d4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=39.4 K 2024-12-04T06:40:52,122 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040ce368a2fe41828382fdfbad2478d4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449915 2024-12-04T06:40:52,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:52,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:52,123 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:52,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:52,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:52,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:52,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:52,124 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5099e56518541e0a04628298270413a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733294450033 2024-12-04T06:40:52,124 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac45369e1b644459cdfde4b191106fa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:52,125 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:52,125 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:52,125 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,125 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8d2d07d2e07a4c449d3ac24ab207251d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=37.0 K 2024-12-04T06:40:52,126 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2d07d2e07a4c449d3ac24ab207251d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449915 2024-12-04T06:40:52,127 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb3ce356029436485e674a3bffa8268, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733294450033 2024-12-04T06:40:52,127 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d638eb0d75b748738f2ca7fa1a2e99dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:52,147 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:52,148 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f04c2968f3b74b4993daac770d5a419f is 50, key is test_row_0/A:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:52,161 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:52,162 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/e40698242bcd4a63b9d6e71cd62e3bd5 is 50, key is test_row_0/B:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:52,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741933_1109 (size=13391) 2024-12-04T06:40:52,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741934_1110 (size=13391) 2024-12-04T06:40:52,227 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f04c2968f3b74b4993daac770d5a419f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f04c2968f3b74b4993daac770d5a419f 2024-12-04T06:40:52,240 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into f04c2968f3b74b4993daac770d5a419f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:52,240 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:52,240 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294452117; duration=0sec 2024-12-04T06:40:52,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:52,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:52,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:52,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:52,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:52,243 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,243 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0dfbd1df5e1348ed95dc4853d30dd655, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=37.0 K 2024-12-04T06:40:52,244 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dfbd1df5e1348ed95dc4853d30dd655, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733294449915 2024-12-04T06:40:52,245 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf9bffdda8d0491d8e4758431fcf7650, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733294450033 2024-12-04T06:40:52,246 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d969d9c90e4e8e9d8219ed5e2561d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:52,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-04T06:40:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,250 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:52,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/db002d4802164b4b87ddfaeb020cfa03 is 50, key is test_row_0/A:col10/1733294451149/Put/seqid=0 2024-12-04T06:40:52,267 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:52,268 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e388db895b3e4ba6be09b8587bedccc0 is 50, key is test_row_0/C:col10/1733294450436/Put/seqid=0 2024-12-04T06:40:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741935_1111 (size=12301) 2024-12-04T06:40:52,288 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/db002d4802164b4b87ddfaeb020cfa03 2024-12-04T06:40:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741936_1112 (size=13391) 2024-12-04T06:40:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/0041e884e2634fb89123c188ed7388a3 is 50, key is test_row_0/B:col10/1733294451149/Put/seqid=0 2024-12-04T06:40:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:52,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:52,312 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/e388db895b3e4ba6be09b8587bedccc0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e388db895b3e4ba6be09b8587bedccc0 2024-12-04T06:40:52,325 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into e388db895b3e4ba6be09b8587bedccc0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:52,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:52,325 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=13, startTime=1733294452123; duration=0sec 2024-12-04T06:40:52,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:52,325 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294512347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294512349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741937_1113 (size=12301) 2024-12-04T06:40:52,357 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/0041e884e2634fb89123c188ed7388a3 2024-12-04T06:40:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294512355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/482e4d955dcf47db987c4359010866be is 50, key is test_row_0/C:col10/1733294451149/Put/seqid=0 2024-12-04T06:40:52,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741938_1114 (size=12301) 2024-12-04T06:40:52,400 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/482e4d955dcf47db987c4359010866be 2024-12-04T06:40:52,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/db002d4802164b4b87ddfaeb020cfa03 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03 2024-12-04T06:40:52,422 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T06:40:52,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/0041e884e2634fb89123c188ed7388a3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3 2024-12-04T06:40:52,439 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T06:40:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/482e4d955dcf47db987c4359010866be as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be 2024-12-04T06:40:52,451 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be, entries=150, sequenceid=440, filesize=12.0 K 2024-12-04T06:40:52,453 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c8a5d67afecdb7afe2ddb08e0bd45184 in 204ms, sequenceid=440, compaction requested=false 2024-12-04T06:40:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-04T06:40:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-04T06:40:52,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T06:40:52,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2930 sec 2024-12-04T06:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:52,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:52,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:52,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3030 sec 2024-12-04T06:40:52,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5974220fb9d54e57a09b3a20bb5b13dc is 50, key is test_row_0/A:col10/1733294452334/Put/seqid=0 2024-12-04T06:40:52,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294512496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294512498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294512499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741939_1115 (size=14741) 2024-12-04T06:40:52,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5974220fb9d54e57a09b3a20bb5b13dc 2024-12-04T06:40:52,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/66879a5ff2994fdfb1f2fde89279e3d5 is 50, key is test_row_0/B:col10/1733294452334/Put/seqid=0 2024-12-04T06:40:52,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741940_1116 (size=12301) 2024-12-04T06:40:52,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294512606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294512606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294512606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,619 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/e40698242bcd4a63b9d6e71cd62e3bd5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/e40698242bcd4a63b9d6e71cd62e3bd5 2024-12-04T06:40:52,625 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into e40698242bcd4a63b9d6e71cd62e3bd5(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:52,625 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:52,626 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294452123; duration=0sec 2024-12-04T06:40:52,626 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:52,626 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:52,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294512817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294512817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294512817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:52,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/66879a5ff2994fdfb1f2fde89279e3d5 2024-12-04T06:40:52,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/810f0c1c8b324ee68d56379da7166fd0 is 50, key is test_row_0/C:col10/1733294452334/Put/seqid=0 2024-12-04T06:40:52,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741941_1117 (size=12301) 2024-12-04T06:40:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294513137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294513137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294513137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T06:40:53,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:53,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-04T06:40:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T06:40:53,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:53,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:53,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:53,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/810f0c1c8b324ee68d56379da7166fd0 2024-12-04T06:40:53,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5974220fb9d54e57a09b3a20bb5b13dc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc 2024-12-04T06:40:53,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc, entries=200, sequenceid=459, filesize=14.4 K 2024-12-04T06:40:53,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/66879a5ff2994fdfb1f2fde89279e3d5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5 2024-12-04T06:40:53,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5, entries=150, sequenceid=459, filesize=12.0 K 2024-12-04T06:40:53,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/810f0c1c8b324ee68d56379da7166fd0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0 2024-12-04T06:40:53,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0, entries=150, sequenceid=459, filesize=12.0 K 2024-12-04T06:40:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T06:40:53,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c8a5d67afecdb7afe2ddb08e0bd45184 in 961ms, sequenceid=459, compaction requested=true 2024-12-04T06:40:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:53,432 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:53,432 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:53,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:53,433 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:53,433 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:53,433 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:53,434 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:53,434 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:53,434 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:53,434 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f04c2968f3b74b4993daac770d5a419f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=39.5 K 2024-12-04T06:40:53,434 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/e40698242bcd4a63b9d6e71cd62e3bd5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=37.1 K 2024-12-04T06:40:53,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:53,434 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f04c2968f3b74b4993daac770d5a419f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:53,435 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e40698242bcd4a63b9d6e71cd62e3bd5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:53,435 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting db002d4802164b4b87ddfaeb020cfa03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733294451122 2024-12-04T06:40:53,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:53,435 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5974220fb9d54e57a09b3a20bb5b13dc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294452328 2024-12-04T06:40:53,435 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0041e884e2634fb89123c188ed7388a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733294451122 2024-12-04T06:40:53,436 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 66879a5ff2994fdfb1f2fde89279e3d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294452328 2024-12-04T06:40:53,454 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:53,455 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/25cccaf1878246b2895397bd75163643 is 50, key is test_row_0/A:col10/1733294452334/Put/seqid=0 2024-12-04T06:40:53,467 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:53,468 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ee3acf23df994c64903cf98313c91dac is 50, key is test_row_0/B:col10/1733294452334/Put/seqid=0 2024-12-04T06:40:53,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-04T06:40:53,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:53,475 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e851d00317e34056b1c6a8c416833d92 is 50, key is test_row_0/A:col10/1733294452497/Put/seqid=0 2024-12-04T06:40:53,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741942_1118 (size=13493) 2024-12-04T06:40:53,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741943_1119 (size=13493) 2024-12-04T06:40:53,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741944_1120 (size=9857) 2024-12-04T06:40:53,519 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e851d00317e34056b1c6a8c416833d92 2024-12-04T06:40:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4d3024b6024c445a94e4126cadd6d608 is 50, key is test_row_0/B:col10/1733294452497/Put/seqid=0 2024-12-04T06:40:53,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741945_1121 (size=9857) 2024-12-04T06:40:53,571 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4d3024b6024c445a94e4126cadd6d608 2024-12-04T06:40:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fa70feed74e849a493bf6a1f4bda4d74 is 50, key is test_row_0/C:col10/1733294452497/Put/seqid=0 2024-12-04T06:40:53,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741946_1122 (size=9857) 2024-12-04T06:40:53,600 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fa70feed74e849a493bf6a1f4bda4d74 2024-12-04T06:40:53,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/e851d00317e34056b1c6a8c416833d92 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92 2024-12-04T06:40:53,618 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92, entries=100, sequenceid=474, filesize=9.6 K 2024-12-04T06:40:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4d3024b6024c445a94e4126cadd6d608 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608 2024-12-04T06:40:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T06:40:53,640 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608, entries=100, sequenceid=474, filesize=9.6 K 2024-12-04T06:40:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/fa70feed74e849a493bf6a1f4bda4d74 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74 2024-12-04T06:40:53,649 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74, entries=100, sequenceid=474, filesize=9.6 K 2024-12-04T06:40:53,650 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for c8a5d67afecdb7afe2ddb08e0bd45184 in 176ms, sequenceid=474, compaction requested=true 2024-12-04T06:40:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-04T06:40:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-04T06:40:53,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T06:40:53,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 331 msec 2024-12-04T06:40:53,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 339 msec 2024-12-04T06:40:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:40:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:53,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f9de2fabf50844639995cf7cf21b4bf2 is 50, key is test_row_0/A:col10/1733294453666/Put/seqid=0 2024-12-04T06:40:53,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741947_1123 (size=12301) 2024-12-04T06:40:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294513750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294513751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294513758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294513888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294513888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294513888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:53,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:53,908 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/25cccaf1878246b2895397bd75163643 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/25cccaf1878246b2895397bd75163643 2024-12-04T06:40:53,911 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ee3acf23df994c64903cf98313c91dac as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ee3acf23df994c64903cf98313c91dac 2024-12-04T06:40:53,922 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into ee3acf23df994c64903cf98313c91dac(size=13.2 K), total size for store is 22.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:53,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:53,923 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294453432; duration=0sec 2024-12-04T06:40:53,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:53,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:53,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:53,929 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:53,929 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:53,929 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:53,929 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e388db895b3e4ba6be09b8587bedccc0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=46.7 K 2024-12-04T06:40:53,933 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e388db895b3e4ba6be09b8587bedccc0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733294450436 2024-12-04T06:40:53,933 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 482e4d955dcf47db987c4359010866be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1733294451122 2024-12-04T06:40:53,933 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 810f0c1c8b324ee68d56379da7166fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294452328 2024-12-04T06:40:53,934 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fa70feed74e849a493bf6a1f4bda4d74, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733294452495 2024-12-04T06:40:53,936 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 25cccaf1878246b2895397bd75163643(size=13.2 K), total size for store is 22.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:53,936 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:53,937 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294453432; duration=0sec 2024-12-04T06:40:53,937 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:53,938 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:53,945 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:53,946 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a857c183aa78483c81e485b2752a236e is 50, key is test_row_0/C:col10/1733294452497/Put/seqid=0 2024-12-04T06:40:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-04T06:40:53,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:53,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:40:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-04T06:40:53,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:40:53,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:40:53,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:40:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T06:40:53,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741948_1124 (size=13527) 2024-12-04T06:40:53,986 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/a857c183aa78483c81e485b2752a236e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a857c183aa78483c81e485b2752a236e 2024-12-04T06:40:53,993 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into a857c183aa78483c81e485b2752a236e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:53,993 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:53,993 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=12, startTime=1733294453432; duration=0sec 2024-12-04T06:40:53,993 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:53,993 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T06:40:54,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f9de2fabf50844639995cf7cf21b4bf2 2024-12-04T06:40:54,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-04T06:40:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:54,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294514108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6686d3b15f8149d59fc401b7dca5815d is 50, key is test_row_0/B:col10/1733294453666/Put/seqid=0 2024-12-04T06:40:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294514108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294514108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1733294514137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,144 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36752 deadline: 1733294514157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,161 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in c8a5d67afecdb7afe2ddb08e0bd45184 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:40:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741949_1125 (size=12301) 2024-12-04T06:40:54,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6686d3b15f8149d59fc401b7dca5815d 2024-12-04T06:40:54,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8f3930cd07354a648d8a16dd81635049 is 50, key is test_row_0/C:col10/1733294453666/Put/seqid=0 2024-12-04T06:40:54,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741950_1126 (size=12301) 2024-12-04T06:40:54,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8f3930cd07354a648d8a16dd81635049 2024-12-04T06:40:54,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/f9de2fabf50844639995cf7cf21b4bf2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2 2024-12-04T06:40:54,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2, entries=150, sequenceid=486, filesize=12.0 K 2024-12-04T06:40:54,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/6686d3b15f8149d59fc401b7dca5815d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d 2024-12-04T06:40:54,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d, entries=150, sequenceid=486, filesize=12.0 K 2024-12-04T06:40:54,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/8f3930cd07354a648d8a16dd81635049 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049 2024-12-04T06:40:54,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-04T06:40:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T06:40:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:40:54,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049, entries=150, sequenceid=486, filesize=12.0 K 2024-12-04T06:40:54,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c8a5d67afecdb7afe2ddb08e0bd45184 in 606ms, sequenceid=486, compaction requested=true 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:54,274 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:54,274 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:54,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:54,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:54,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:54,276 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,276 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ee3acf23df994c64903cf98313c91dac, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=34.8 K 2024-12-04T06:40:54,276 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:54,276 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:54,276 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,276 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/25cccaf1878246b2895397bd75163643, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=34.8 K 2024-12-04T06:40:54,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3acf23df994c64903cf98313c91dac, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294452328 2024-12-04T06:40:54,277 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25cccaf1878246b2895397bd75163643, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294452328 2024-12-04T06:40:54,277 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3024b6024c445a94e4126cadd6d608, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733294452495 2024-12-04T06:40:54,277 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6686d3b15f8149d59fc401b7dca5815d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733294453660 2024-12-04T06:40:54,277 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e851d00317e34056b1c6a8c416833d92, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733294452495 2024-12-04T06:40:54,280 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9de2fabf50844639995cf7cf21b4bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733294453660 2024-12-04T06:40:54,292 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:54,293 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/5194f3fba92f4f9ab3ef41832486cbcb is 50, key is test_row_0/B:col10/1733294453666/Put/seqid=0 2024-12-04T06:40:54,298 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:54,298 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5ed3e9fe96bc49409bd526976d2a9b21 is 50, key is test_row_0/A:col10/1733294453666/Put/seqid=0 2024-12-04T06:40:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741951_1127 (size=13595) 2024-12-04T06:40:54,343 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/5194f3fba92f4f9ab3ef41832486cbcb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/5194f3fba92f4f9ab3ef41832486cbcb 2024-12-04T06:40:54,353 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into 5194f3fba92f4f9ab3ef41832486cbcb(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:54,353 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:54,353 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294454274; duration=0sec 2024-12-04T06:40:54,353 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:54,353 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:54,354 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T06:40:54,355 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-04T06:40:54,355 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-04T06:40:54,355 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. because compaction request was cancelled 2024-12-04T06:40:54,355 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:54,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741952_1128 (size=13595) 2024-12-04T06:40:54,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-04T06:40:54,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,421 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:54,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. as already flushing 2024-12-04T06:40:54,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/50b08216716443a9b14af6f55bdf1893 is 50, key is test_row_0/A:col10/1733294453744/Put/seqid=0 2024-12-04T06:40:54,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294514453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294514455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294514456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741953_1129 (size=14741) 2024-12-04T06:40:54,470 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/50b08216716443a9b14af6f55bdf1893 2024-12-04T06:40:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4e8a9ed2e0894d60987f5275133e477c is 50, key is test_row_0/B:col10/1733294453744/Put/seqid=0 2024-12-04T06:40:54,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741954_1130 (size=12301) 2024-12-04T06:40:54,500 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4e8a9ed2e0894d60987f5275133e477c 2024-12-04T06:40:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/60ffdbfb0d16436f865fcd4e35cf917e is 50, key is test_row_0/C:col10/1733294453744/Put/seqid=0 2024-12-04T06:40:54,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741955_1131 (size=12301) 2024-12-04T06:40:54,524 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/60ffdbfb0d16436f865fcd4e35cf917e 2024-12-04T06:40:54,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/50b08216716443a9b14af6f55bdf1893 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893 2024-12-04T06:40:54,564 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893, entries=200, sequenceid=511, filesize=14.4 K 2024-12-04T06:40:54,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/4e8a9ed2e0894d60987f5275133e477c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c 2024-12-04T06:40:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294514566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294514566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,571 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c, entries=150, sequenceid=511, filesize=12.0 K 2024-12-04T06:40:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294514566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/60ffdbfb0d16436f865fcd4e35cf917e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e 2024-12-04T06:40:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T06:40:54,583 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e, entries=150, sequenceid=511, filesize=12.0 K 2024-12-04T06:40:54,585 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c8a5d67afecdb7afe2ddb08e0bd45184 in 164ms, sequenceid=511, compaction requested=true 2024-12-04T06:40:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-04T06:40:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-04T06:40:54,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T06:40:54,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 632 msec 2024-12-04T06:40:54,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 643 msec 2024-12-04T06:40:54,768 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/5ed3e9fe96bc49409bd526976d2a9b21 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ed3e9fe96bc49409bd526976d2a9b21 2024-12-04T06:40:54,774 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 5ed3e9fe96bc49409bd526976d2a9b21(size=13.3 K), total size for store is 27.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:54,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:54,775 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294454274; duration=0sec 2024-12-04T06:40:54,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:54,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:54,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:40:54,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/02fd7fd94ff94bd884d40868f1e544d6 is 50, key is test_row_0/A:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741956_1132 (size=17181) 2024-12-04T06:40:54,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/02fd7fd94ff94bd884d40868f1e544d6 2024-12-04T06:40:54,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d15ffccee6894aeabbd3b2fb356824d9 is 50, key is test_row_0/B:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:54,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741957_1133 (size=12301) 2024-12-04T06:40:54,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d15ffccee6894aeabbd3b2fb356824d9 2024-12-04T06:40:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294514822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294514833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294514835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/6ba873a7cb9f4491a9d22e39620281b6 is 50, key is test_row_0/C:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741958_1134 (size=12301) 2024-12-04T06:40:54,879 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-04T06:40:54,879 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:54,881 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-04T06:40:54,881 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:54,882 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-04T06:40:54,882 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:54,885 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-04T06:40:54,885 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:54,885 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:54,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:54,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:54,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:54,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294514947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294514947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294514947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T06:40:55,087 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:40:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36768 deadline: 1733294515157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36748 deadline: 1733294515157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1733294515157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:40:55,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/6ba873a7cb9f4491a9d22e39620281b6 2024-12-04T06:40:55,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/02fd7fd94ff94bd884d40868f1e544d6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6 2024-12-04T06:40:55,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6, entries=250, sequenceid=526, filesize=16.8 K 2024-12-04T06:40:55,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/d15ffccee6894aeabbd3b2fb356824d9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9 2024-12-04T06:40:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9, entries=150, sequenceid=526, filesize=12.0 K 2024-12-04T06:40:55,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/6ba873a7cb9f4491a9d22e39620281b6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6 2024-12-04T06:40:55,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6, entries=150, sequenceid=526, filesize=12.0 K 2024-12-04T06:40:55,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c8a5d67afecdb7afe2ddb08e0bd45184 in 484ms, sequenceid=526, compaction requested=true 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:55,268 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:55,268 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8a5d67afecdb7afe2ddb08e0bd45184:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:40:55,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:55,269 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:55,269 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:40:55,269 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/A is initiating minor compaction (all files) 2024-12-04T06:40:55,269 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/B is initiating minor compaction (all files) 2024-12-04T06:40:55,269 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/A in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:55,269 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/B in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:55,269 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ed3e9fe96bc49409bd526976d2a9b21, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=44.5 K 2024-12-04T06:40:55,269 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/5194f3fba92f4f9ab3ef41832486cbcb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=37.3 K 2024-12-04T06:40:55,270 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed3e9fe96bc49409bd526976d2a9b21, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733294453660 2024-12-04T06:40:55,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5194f3fba92f4f9ab3ef41832486cbcb, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733294453660 2024-12-04T06:40:55,270 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b08216716443a9b14af6f55bdf1893, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733294453741 2024-12-04T06:40:55,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8a9ed2e0894d60987f5275133e477c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733294453741 2024-12-04T06:40:55,271 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02fd7fd94ff94bd884d40868f1e544d6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733294454435 2024-12-04T06:40:55,271 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d15ffccee6894aeabbd3b2fb356824d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733294454435 2024-12-04T06:40:55,278 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#B#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:55,279 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ac2ae4337a7443fd94af158334c240d5 is 50, key is test_row_0/B:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:55,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741959_1135 (size=13697) 2024-12-04T06:40:55,305 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#A#compaction#123 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:55,305 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/255542a3cf424276b3e1a550690af35d is 50, key is test_row_0/A:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:55,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741960_1136 (size=13697) 2024-12-04T06:40:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:40:55,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T06:40:55,469 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-04T06:40:55,469 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-04T06:40:55,469 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:55,469 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:55,469 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,469 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:40:55,469 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,469 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:40:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:40:55,469 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-04T06:40:55,469 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:40:55,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:55,470 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,470 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:40:55,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:55,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:40:55,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/969b311f4f9847428f7ac51c71af3194 is 50, key is test_row_0/A:col10/1733294455467/Put/seqid=0 2024-12-04T06:40:55,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741961_1137 (size=12301) 2024-12-04T06:40:55,690 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/ac2ae4337a7443fd94af158334c240d5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ac2ae4337a7443fd94af158334c240d5 2024-12-04T06:40:55,695 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/B of c8a5d67afecdb7afe2ddb08e0bd45184 into ac2ae4337a7443fd94af158334c240d5(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:55,695 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:55,695 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/B, priority=13, startTime=1733294455268; duration=0sec 2024-12-04T06:40:55,696 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:40:55,696 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:B 2024-12-04T06:40:55,696 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:40:55,697 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:40:55,697 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): c8a5d67afecdb7afe2ddb08e0bd45184/C is initiating minor compaction (all files) 2024-12-04T06:40:55,697 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8a5d67afecdb7afe2ddb08e0bd45184/C in TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:40:55,697 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a857c183aa78483c81e485b2752a236e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp, totalSize=49.2 K 2024-12-04T06:40:55,697 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a857c183aa78483c81e485b2752a236e, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1733294452328 2024-12-04T06:40:55,698 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f3930cd07354a648d8a16dd81635049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1733294453660 2024-12-04T06:40:55,698 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ffdbfb0d16436f865fcd4e35cf917e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1733294453741 2024-12-04T06:40:55,698 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba873a7cb9f4491a9d22e39620281b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733294454435 2024-12-04T06:40:55,713 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8a5d67afecdb7afe2ddb08e0bd45184#C#compaction#125 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:40:55,713 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/909797f3e06f4e61be14035958c2041b is 50, key is test_row_0/C:col10/1733294454783/Put/seqid=0 2024-12-04T06:40:55,715 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/255542a3cf424276b3e1a550690af35d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/255542a3cf424276b3e1a550690af35d 2024-12-04T06:40:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741962_1138 (size=13663) 2024-12-04T06:40:55,722 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/A of c8a5d67afecdb7afe2ddb08e0bd45184 into 255542a3cf424276b3e1a550690af35d(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:55,722 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:55,722 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/A, priority=13, startTime=1733294455268; duration=0sec 2024-12-04T06:40:55,722 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:55,723 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:A 2024-12-04T06:40:55,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/969b311f4f9847428f7ac51c71af3194 2024-12-04T06:40:55,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7d11bc2e37bb4798b7caa44e48b35497 is 50, key is test_row_0/B:col10/1733294455467/Put/seqid=0 2024-12-04T06:40:55,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741963_1139 (size=12301) 2024-12-04T06:40:56,124 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/909797f3e06f4e61be14035958c2041b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/909797f3e06f4e61be14035958c2041b 2024-12-04T06:40:56,129 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8a5d67afecdb7afe2ddb08e0bd45184/C of c8a5d67afecdb7afe2ddb08e0bd45184 into 909797f3e06f4e61be14035958c2041b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:40:56,129 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:56,129 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., storeName=c8a5d67afecdb7afe2ddb08e0bd45184/C, priority=12, startTime=1733294455268; duration=0sec 2024-12-04T06:40:56,129 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:40:56,129 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8a5d67afecdb7afe2ddb08e0bd45184:C 2024-12-04T06:40:56,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7d11bc2e37bb4798b7caa44e48b35497 2024-12-04T06:40:56,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d389b339d6a74365a01fa0907791da86 is 50, key is test_row_0/C:col10/1733294455467/Put/seqid=0 2024-12-04T06:40:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741964_1140 (size=12301) 2024-12-04T06:40:56,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d389b339d6a74365a01fa0907791da86 2024-12-04T06:40:56,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/969b311f4f9847428f7ac51c71af3194 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/969b311f4f9847428f7ac51c71af3194 2024-12-04T06:40:56,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/969b311f4f9847428f7ac51c71af3194, entries=150, sequenceid=547, filesize=12.0 K 2024-12-04T06:40:56,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/7d11bc2e37bb4798b7caa44e48b35497 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7d11bc2e37bb4798b7caa44e48b35497 2024-12-04T06:40:56,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7d11bc2e37bb4798b7caa44e48b35497, entries=150, sequenceid=547, filesize=12.0 K 2024-12-04T06:40:56,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/d389b339d6a74365a01fa0907791da86 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d389b339d6a74365a01fa0907791da86 2024-12-04T06:40:56,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d389b339d6a74365a01fa0907791da86, entries=150, sequenceid=547, filesize=12.0 K 2024-12-04T06:40:56,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for c8a5d67afecdb7afe2ddb08e0bd45184 in 1257ms, sequenceid=547, compaction requested=false 2024-12-04T06:40:56,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8a5d67afecdb7afe2ddb08e0bd45184: 2024-12-04T06:40:59,080 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:41:04,169 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-04T06:41:04,169 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:04,169 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,169 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:04,230 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-04T06:41:04,230 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:04,230 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,230 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-04T06:41:04,230 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2512 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2442 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1147 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3435 rows 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1145 2024-12-04T06:41:04,231 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3426 rows 2024-12-04T06:41:04,231 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:41:04,231 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:41:04,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:04,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:04,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:41:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:04,248 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294464247"}]},"ts":"1733294464247"} 2024-12-04T06:41:04,251 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:41:04,251 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:41:04,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:41:04,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, UNASSIGN}] 2024-12-04T06:41:04,260 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, UNASSIGN 2024-12-04T06:41:04,262 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45619 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=607fd5c6574c,38257,1733294430862, table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T06:41:04,262 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c8a5d67afecdb7afe2ddb08e0bd45184, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:04,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, UNASSIGN because future has completed 2024-12-04T06:41:04,266 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:41:04,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:04,425 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:41:04,425 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing c8a5d67afecdb7afe2ddb08e0bd45184, disabling compactions & flushes 2024-12-04T06:41:04,426 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. after waiting 0 ms 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:41:04,426 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing c8a5d67afecdb7afe2ddb08e0bd45184 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=A 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=B 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8a5d67afecdb7afe2ddb08e0bd45184, store=C 2024-12-04T06:41:04,426 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:04,431 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/456c770eda714b61a474e0c7656bee37 is 50, key is test_row_0/A:col10/1733294464228/Put/seqid=0 2024-12-04T06:41:04,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741965_1141 (size=9857) 2024-12-04T06:41:04,441 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/456c770eda714b61a474e0c7656bee37 2024-12-04T06:41:04,450 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/384c3ff764914d218499f3f750d699e9 is 50, key is test_row_0/B:col10/1733294464228/Put/seqid=0 2024-12-04T06:41:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741966_1142 (size=9857) 2024-12-04T06:41:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:04,858 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/384c3ff764914d218499f3f750d699e9 2024-12-04T06:41:04,866 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2658a46e56c041e2b4012cb74a3be1aa is 50, key is test_row_0/C:col10/1733294464228/Put/seqid=0 2024-12-04T06:41:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741967_1143 (size=9857) 2024-12-04T06:41:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:05,276 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2658a46e56c041e2b4012cb74a3be1aa 2024-12-04T06:41:05,282 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/A/456c770eda714b61a474e0c7656bee37 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/456c770eda714b61a474e0c7656bee37 2024-12-04T06:41:05,286 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/456c770eda714b61a474e0c7656bee37, entries=100, sequenceid=555, filesize=9.6 K 2024-12-04T06:41:05,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/B/384c3ff764914d218499f3f750d699e9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/384c3ff764914d218499f3f750d699e9 2024-12-04T06:41:05,291 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/384c3ff764914d218499f3f750d699e9, entries=100, sequenceid=555, filesize=9.6 K 2024-12-04T06:41:05,292 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/.tmp/C/2658a46e56c041e2b4012cb74a3be1aa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2658a46e56c041e2b4012cb74a3be1aa 2024-12-04T06:41:05,296 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2658a46e56c041e2b4012cb74a3be1aa, entries=100, sequenceid=555, filesize=9.6 K 2024-12-04T06:41:05,297 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c8a5d67afecdb7afe2ddb08e0bd45184 in 871ms, sequenceid=555, compaction requested=true 2024-12-04T06:41:05,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/d261b12f97a54ba5a94db08292b3553a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/2c4ca276f532431d8d5822e2d97c7903, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5d1954339bcb4ceda72ecab6d6351d1d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/fb01dd09c8d44fe6b1c36aa786680dfa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a18b493299484087ad0a967ca91faca4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/b4dad0b7eae74378813c9446ac111a3f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3d57c792b81c4949b6cf548566609961, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/77a65b8f98f94861b16e1ae72ac51e19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/040ce368a2fe41828382fdfbad2478d4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f04c2968f3b74b4993daac770d5a419f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/25cccaf1878246b2895397bd75163643, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ed3e9fe96bc49409bd526976d2a9b21, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6] to archive 2024-12-04T06:41:05,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:05,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7722267f671b4a17805670bbd989b72e 2024-12-04T06:41:05,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/80de97402b7e4e3ab1551bb099956ac1 2024-12-04T06:41:05,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ba77019c9a854d958baeeb4263e6de75 2024-12-04T06:41:05,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/d261b12f97a54ba5a94db08292b3553a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/d261b12f97a54ba5a94db08292b3553a 2024-12-04T06:41:05,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/274b3e7cf26d4c99bb8b88c4aef6488d 2024-12-04T06:41:05,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a4b33a1c823446a9b4ccce7505735dde 2024-12-04T06:41:05,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/dc063ccbd9e54afe9a4a1fb88bf53430 2024-12-04T06:41:05,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/2c4ca276f532431d8d5822e2d97c7903 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/2c4ca276f532431d8d5822e2d97c7903 2024-12-04T06:41:05,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/8aa8bce2b26a4dc2beb4b7ec572a3cb3 2024-12-04T06:41:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc4f421f2a9449d191cbfb174ef3988d 2024-12-04T06:41:05,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5d1954339bcb4ceda72ecab6d6351d1d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5d1954339bcb4ceda72ecab6d6351d1d 2024-12-04T06:41:05,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/cc23441645624b82876a3bae3e64d1d9 2024-12-04T06:41:05,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3f1ab0b3df5d41a09e3f0716834c709d 2024-12-04T06:41:05,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/fb01dd09c8d44fe6b1c36aa786680dfa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/fb01dd09c8d44fe6b1c36aa786680dfa 2024-12-04T06:41:05,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7e39d8ba376a4d08aecb8492d3619bc5 2024-12-04T06:41:05,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a18b493299484087ad0a967ca91faca4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/a18b493299484087ad0a967ca91faca4 2024-12-04T06:41:05,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5b0d9d3dd18460cbfc51781cb3bfd59 2024-12-04T06:41:05,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/caace600ca13479094643800ac0fd7cc 2024-12-04T06:41:05,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/551ceb208ea640c693103312f78f89a7 2024-12-04T06:41:05,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/b4dad0b7eae74378813c9446ac111a3f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/b4dad0b7eae74378813c9446ac111a3f 2024-12-04T06:41:05,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/7538316e60cd4614a237989fde5e9f19 2024-12-04T06:41:05,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/ed0b0eead63c4df7ae38e0b02691c155 2024-12-04T06:41:05,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e42922202cb9421cb5775896f3bb0d72 2024-12-04T06:41:05,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3d57c792b81c4949b6cf548566609961 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/3d57c792b81c4949b6cf548566609961 2024-12-04T06:41:05,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/0ce4be02f6774ee58db5c86f24e7b6fa 2024-12-04T06:41:05,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/c6977ed664dd4c7aaca1cdefee592bf8 2024-12-04T06:41:05,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/77a65b8f98f94861b16e1ae72ac51e19 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/77a65b8f98f94861b16e1ae72ac51e19 2024-12-04T06:41:05,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/63c906ebafe0466798d5675266fa5f1f 2024-12-04T06:41:05,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/984690e298774589bfe944cd45d7c03d 2024-12-04T06:41:05,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/040ce368a2fe41828382fdfbad2478d4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/040ce368a2fe41828382fdfbad2478d4 2024-12-04T06:41:05,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f5099e56518541e0a04628298270413a 2024-12-04T06:41:05,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ac45369e1b644459cdfde4b191106fa 2024-12-04T06:41:05,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f04c2968f3b74b4993daac770d5a419f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f04c2968f3b74b4993daac770d5a419f 2024-12-04T06:41:05,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/db002d4802164b4b87ddfaeb020cfa03 2024-12-04T06:41:05,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5974220fb9d54e57a09b3a20bb5b13dc 2024-12-04T06:41:05,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/25cccaf1878246b2895397bd75163643 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/25cccaf1878246b2895397bd75163643 2024-12-04T06:41:05,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/e851d00317e34056b1c6a8c416833d92 2024-12-04T06:41:05,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ed3e9fe96bc49409bd526976d2a9b21 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/5ed3e9fe96bc49409bd526976d2a9b21 2024-12-04T06:41:05,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/f9de2fabf50844639995cf7cf21b4bf2 2024-12-04T06:41:05,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/50b08216716443a9b14af6f55bdf1893 2024-12-04T06:41:05,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/02fd7fd94ff94bd884d40868f1e544d6 2024-12-04T06:41:05,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/a55795bb1224449fae1fa2b7b7bd55e6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/14240f60b2d141a4805886c81531b485, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8ad4386e7b134f55958a545cc032dec1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0c9c9bf8e40a40309d58bcea9b3f6169, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/b8fb083f043c499591218408e7b0c7e6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ca87f877b6dd442e880c78fd331e7560, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/498719cf8d4e4fd0a5edc41b4be74b9c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/cde165af65bc4333b9c5d027d76f0fda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8d2d07d2e07a4c449d3ac24ab207251d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/e40698242bcd4a63b9d6e71cd62e3bd5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ee3acf23df994c64903cf98313c91dac, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/5194f3fba92f4f9ab3ef41832486cbcb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9] to archive 2024-12-04T06:41:05,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:05,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/3ce1f65ae7584d10bd4e18b65cec8cff 2024-12-04T06:41:05,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4362678d468140c09821bf6f5efaf895 2024-12-04T06:41:05,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/a55795bb1224449fae1fa2b7b7bd55e6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/a55795bb1224449fae1fa2b7b7bd55e6 2024-12-04T06:41:05,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/67a82d5b0b23402589f71b0474bd60a6 2024-12-04T06:41:05,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2690b41d255a460cb1c27d4ff68feeda 2024-12-04T06:41:05,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8c6643705d1b4ee698f9cc65c65df5cb 2024-12-04T06:41:05,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/14240f60b2d141a4805886c81531b485 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/14240f60b2d141a4805886c81531b485 2024-12-04T06:41:05,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/148e839daa6a469e836a30b6ee01cced 2024-12-04T06:41:05,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/964aa27388064c15a0810cd7e56290eb 2024-12-04T06:41:05,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8ad4386e7b134f55958a545cc032dec1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8ad4386e7b134f55958a545cc032dec1 2024-12-04T06:41:05,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/010fdbb61d654376ad5cd2e0f5eb743e 2024-12-04T06:41:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:05,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d4527d5c578f402a80ad88aa0415d3c5 2024-12-04T06:41:05,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0c9c9bf8e40a40309d58bcea9b3f6169 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0c9c9bf8e40a40309d58bcea9b3f6169 2024-12-04T06:41:05,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2cedc7a9495940d8ba78fc55f0e4bcb8 2024-12-04T06:41:05,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d101ce9c918f4748aee27361f196a174 2024-12-04T06:41:05,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/b8fb083f043c499591218408e7b0c7e6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/b8fb083f043c499591218408e7b0c7e6 2024-12-04T06:41:05,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/320bbafaba8543dbb721fbc50d7e1bc2 2024-12-04T06:41:05,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7f81394b84e64348b2c2c90438f578ac 2024-12-04T06:41:05,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ca87f877b6dd442e880c78fd331e7560 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ca87f877b6dd442e880c78fd331e7560 2024-12-04T06:41:05,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/88bb7f61134f47fcb121d6568b1c0328 2024-12-04T06:41:05,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/74964120ffec42cf8644381a6a5100c2 2024-12-04T06:41:05,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/bb7e05cacd464c4c899dd4e4f5804b7b 2024-12-04T06:41:05,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/498719cf8d4e4fd0a5edc41b4be74b9c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/498719cf8d4e4fd0a5edc41b4be74b9c 2024-12-04T06:41:05,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6788d0820284466d9898ade9ba4151a0 2024-12-04T06:41:05,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/781b48d1892c40c5af20160b18fce753 2024-12-04T06:41:05,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/cde165af65bc4333b9c5d027d76f0fda to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/cde165af65bc4333b9c5d027d76f0fda 2024-12-04T06:41:05,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/2247cc126fc54c918524b274872844eb 2024-12-04T06:41:05,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/fe1d6b224a574458ab791c0b075f6cda 2024-12-04T06:41:05,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8d2d07d2e07a4c449d3ac24ab207251d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8d2d07d2e07a4c449d3ac24ab207251d 2024-12-04T06:41:05,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/04ea5a14ae8a43b6b5d08bb80e394657 2024-12-04T06:41:05,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/8fb3ce356029436485e674a3bffa8268 2024-12-04T06:41:05,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/e40698242bcd4a63b9d6e71cd62e3bd5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/e40698242bcd4a63b9d6e71cd62e3bd5 2024-12-04T06:41:05,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d638eb0d75b748738f2ca7fa1a2e99dc 2024-12-04T06:41:05,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/0041e884e2634fb89123c188ed7388a3 2024-12-04T06:41:05,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ee3acf23df994c64903cf98313c91dac to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ee3acf23df994c64903cf98313c91dac 2024-12-04T06:41:05,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/66879a5ff2994fdfb1f2fde89279e3d5 2024-12-04T06:41:05,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4d3024b6024c445a94e4126cadd6d608 2024-12-04T06:41:05,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/5194f3fba92f4f9ab3ef41832486cbcb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/5194f3fba92f4f9ab3ef41832486cbcb 2024-12-04T06:41:05,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/6686d3b15f8149d59fc401b7dca5815d 2024-12-04T06:41:05,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/4e8a9ed2e0894d60987f5275133e477c 2024-12-04T06:41:05,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/d15ffccee6894aeabbd3b2fb356824d9 2024-12-04T06:41:05,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/5243debfd76d45bdac14b20066af3cfa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7740626659ac4e239c69cbf2ea715583, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/3b7a140313134d2198a04737720f0daa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/16d5e46facbc4c6995db4936a131bcee, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/27710b3d986b4ce6a9ee7b832877a29e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/86d3cea0df1e4e14beef1754ee49a1e9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1731cbb322bd41fd9538a4fb87a52f7e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/330575ebd1e74b4fbea75ac3158eacaf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0dfbd1df5e1348ed95dc4853d30dd655, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e388db895b3e4ba6be09b8587bedccc0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a857c183aa78483c81e485b2752a236e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6] to archive 2024-12-04T06:41:05,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:05,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/b022948605da4521b7953fdd2b0d42ad 2024-12-04T06:41:05,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/dc8d85c360a145cb8b54ea1fb0d15831 2024-12-04T06:41:05,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/5243debfd76d45bdac14b20066af3cfa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/5243debfd76d45bdac14b20066af3cfa 2024-12-04T06:41:05,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e36bddb088614384a9c4b01cf16c9591 2024-12-04T06:41:05,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/94116d2ff86443589a5ed6cf6a77b527 2024-12-04T06:41:05,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a01ff9af9b084ccbb85b0d795f5ea588 2024-12-04T06:41:05,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7740626659ac4e239c69cbf2ea715583 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7740626659ac4e239c69cbf2ea715583 2024-12-04T06:41:05,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/85459510132f4047b388685bc5ec2ec0 2024-12-04T06:41:05,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fba657e3d51a4346bcd158f9c5582535 2024-12-04T06:41:05,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/3b7a140313134d2198a04737720f0daa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/3b7a140313134d2198a04737720f0daa 2024-12-04T06:41:05,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a766c4109ef64882a30e76c04223d575 2024-12-04T06:41:05,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/f98fe68f049e4f2eb2623884b1a54107 2024-12-04T06:41:05,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/16d5e46facbc4c6995db4936a131bcee to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/16d5e46facbc4c6995db4936a131bcee 2024-12-04T06:41:05,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2a57a8b4657e4f76b18cbc4f1d800041 2024-12-04T06:41:05,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e6c2d2ca64784e3da1332281b556c081 2024-12-04T06:41:05,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/27710b3d986b4ce6a9ee7b832877a29e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/27710b3d986b4ce6a9ee7b832877a29e 2024-12-04T06:41:05,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/137bbf761c004ecba7ab42e121645dba 2024-12-04T06:41:05,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7760378fd14e4d58a9693161e77079c4 2024-12-04T06:41:05,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/86d3cea0df1e4e14beef1754ee49a1e9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/86d3cea0df1e4e14beef1754ee49a1e9 2024-12-04T06:41:05,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1b53c006da1243a9b344dee910f154cf 2024-12-04T06:41:05,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8355c788426a4dada0bdbe714bf0494b 2024-12-04T06:41:05,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/7d468837de25424fa53c8f49c2446840 2024-12-04T06:41:05,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1731cbb322bd41fd9538a4fb87a52f7e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/1731cbb322bd41fd9538a4fb87a52f7e 2024-12-04T06:41:05,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/659131fc5dbe41e18aa3ddffee91da07 2024-12-04T06:41:05,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0ae0b43c48d14ebf8da637ca71453b3c 2024-12-04T06:41:05,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/330575ebd1e74b4fbea75ac3158eacaf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/330575ebd1e74b4fbea75ac3158eacaf 2024-12-04T06:41:05,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ae89e573b645450487bef80eba86416c 2024-12-04T06:41:05,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/ec8fb3c1c76841468426bafddc538be2 2024-12-04T06:41:05,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0dfbd1df5e1348ed95dc4853d30dd655 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/0dfbd1df5e1348ed95dc4853d30dd655 2024-12-04T06:41:05,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/49593a026f34460e85683d92c996b6bb 2024-12-04T06:41:05,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/cf9bffdda8d0491d8e4758431fcf7650 2024-12-04T06:41:05,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e388db895b3e4ba6be09b8587bedccc0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/e388db895b3e4ba6be09b8587bedccc0 2024-12-04T06:41:05,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d7d969d9c90e4e8e9d8219ed5e2561d3 2024-12-04T06:41:05,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/482e4d955dcf47db987c4359010866be 2024-12-04T06:41:05,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/810f0c1c8b324ee68d56379da7166fd0 2024-12-04T06:41:05,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a857c183aa78483c81e485b2752a236e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/a857c183aa78483c81e485b2752a236e 2024-12-04T06:41:05,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/fa70feed74e849a493bf6a1f4bda4d74 2024-12-04T06:41:05,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/8f3930cd07354a648d8a16dd81635049 2024-12-04T06:41:05,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/60ffdbfb0d16436f865fcd4e35cf917e 2024-12-04T06:41:05,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/6ba873a7cb9f4491a9d22e39620281b6 2024-12-04T06:41:05,500 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/recovered.edits/558.seqid, newMaxSeqId=558, maxSeqId=1 2024-12-04T06:41:05,501 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. 2024-12-04T06:41:05,501 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for c8a5d67afecdb7afe2ddb08e0bd45184: Waiting for close lock at 1733294464425Running coprocessor pre-close hooks at 1733294464426 (+1 ms)Disabling compacts and flushes for region at 1733294464426Disabling writes for close at 1733294464426Obtaining lock to block concurrent updates at 1733294464426Preparing flush snapshotting stores in c8a5d67afecdb7afe2ddb08e0bd45184 at 1733294464426Finished memstore snapshotting TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733294464426Flushing stores of TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184. at 1733294464427 (+1 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/A: creating writer at 1733294464427Flushing c8a5d67afecdb7afe2ddb08e0bd45184/A: appending metadata at 1733294464431 (+4 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/A: closing flushed file at 1733294464431Flushing c8a5d67afecdb7afe2ddb08e0bd45184/B: creating writer at 1733294464447 (+16 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/B: appending metadata at 1733294464450 (+3 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/B: closing flushed file at 1733294464450Flushing c8a5d67afecdb7afe2ddb08e0bd45184/C: creating writer at 1733294464863 (+413 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/C: appending metadata at 1733294464866 (+3 ms)Flushing c8a5d67afecdb7afe2ddb08e0bd45184/C: closing flushed file at 1733294464866Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ae4b34: reopening flushed file at 1733294465281 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@352a608e: reopening flushed file at 1733294465286 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ff6d0cd: reopening flushed file at 1733294465291 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for c8a5d67afecdb7afe2ddb08e0bd45184 in 871ms, sequenceid=555, compaction requested=true at 1733294465297 (+6 ms)Writing region close event to WAL at 1733294465496 (+199 ms)Running coprocessor post-close hooks at 1733294465501 (+5 ms)Closed at 1733294465501 2024-12-04T06:41:05,504 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:41:05,505 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c8a5d67afecdb7afe2ddb08e0bd45184, regionState=CLOSED 2024-12-04T06:41:05,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:05,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T06:41:05,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure c8a5d67afecdb7afe2ddb08e0bd45184, server=607fd5c6574c,38257,1733294430862 in 1.2430 sec 2024-12-04T06:41:05,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-04T06:41:05,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8a5d67afecdb7afe2ddb08e0bd45184, UNASSIGN in 1.2520 sec 2024-12-04T06:41:05,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T06:41:05,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2620 sec 2024-12-04T06:41:05,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294465518"}]},"ts":"1733294465518"} 2024-12-04T06:41:05,520 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:41:05,520 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:41:05,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2820 sec 2024-12-04T06:41:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T06:41:06,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:06,400 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:41:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,406 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T06:41:06,408 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:41:06,415 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/recovered.edits] 2024-12-04T06:41:06,419 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/255542a3cf424276b3e1a550690af35d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/255542a3cf424276b3e1a550690af35d 2024-12-04T06:41:06,420 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/456c770eda714b61a474e0c7656bee37 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/456c770eda714b61a474e0c7656bee37 2024-12-04T06:41:06,421 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/969b311f4f9847428f7ac51c71af3194 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/A/969b311f4f9847428f7ac51c71af3194 2024-12-04T06:41:06,424 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/384c3ff764914d218499f3f750d699e9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/384c3ff764914d218499f3f750d699e9 2024-12-04T06:41:06,425 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7d11bc2e37bb4798b7caa44e48b35497 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/7d11bc2e37bb4798b7caa44e48b35497 2024-12-04T06:41:06,426 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ac2ae4337a7443fd94af158334c240d5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/B/ac2ae4337a7443fd94af158334c240d5 2024-12-04T06:41:06,429 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2658a46e56c041e2b4012cb74a3be1aa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/2658a46e56c041e2b4012cb74a3be1aa 2024-12-04T06:41:06,430 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/909797f3e06f4e61be14035958c2041b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/909797f3e06f4e61be14035958c2041b 2024-12-04T06:41:06,431 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d389b339d6a74365a01fa0907791da86 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/C/d389b339d6a74365a01fa0907791da86 2024-12-04T06:41:06,433 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/recovered.edits/558.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184/recovered.edits/558.seqid 2024-12-04T06:41:06,434 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/c8a5d67afecdb7afe2ddb08e0bd45184 2024-12-04T06:41:06,434 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:41:06,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T06:41:06,451 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:41:06,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:41:06,485 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,485 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:41:06,485 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294466485"}]},"ts":"9223372036854775807"} 2024-12-04T06:41:06,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:41:06,489 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c8a5d67afecdb7afe2ddb08e0bd45184, NAME => 'TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:41:06,489 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:41:06,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294466489"}]},"ts":"9223372036854775807"} 2024-12-04T06:41:06,492 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:41:06,493 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 92 msec 2024-12-04T06:41:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T06:41:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:41:06,518 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:06,533 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-316354366_22 at /127.0.0.1:44468 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;607fd5c6574c:38257-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=379 (was 337) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7145 (was 7279) 2024-12-04T06:41:06,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=379, ProcessCount=11, AvailableMemoryMB=7144 2024-12-04T06:41:06,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:41:06,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:41:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:06,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:41:06,549 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:06,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-04T06:41:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T06:41:06,550 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:41:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741968_1144 (size=1061) 2024-12-04T06:41:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T06:41:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T06:41:06,959 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:41:06,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741969_1145 (size=53) 2024-12-04T06:41:06,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:41:06,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae9d50f9f78f5f45c22cc72ad2361274, disabling compactions & flushes 2024-12-04T06:41:06,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:06,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:06,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. after waiting 0 ms 2024-12-04T06:41:06,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:06,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:06,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae9d50f9f78f5f45c22cc72ad2361274: Waiting for close lock at 1733294466969Disabling compacts and flushes for region at 1733294466969Disabling writes for close at 1733294466970 (+1 ms)Writing region close event to WAL at 1733294466970Closed at 1733294466970 2024-12-04T06:41:06,971 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:41:06,972 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294466971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294466971"}]},"ts":"1733294466971"} 2024-12-04T06:41:06,977 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:41:06,978 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:41:06,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294466978"}]},"ts":"1733294466978"} 2024-12-04T06:41:06,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:41:06,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, ASSIGN}] 2024-12-04T06:41:06,983 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, ASSIGN 2024-12-04T06:41:06,984 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:41:07,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:07,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, ASSIGN because future has completed 2024-12-04T06:41:07,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T06:41:07,295 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:07,295 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:41:07,296 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,296 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:41:07,296 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,296 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,297 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,299 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:07,299 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName A 2024-12-04T06:41:07,300 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:07,300 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:07,300 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,301 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:07,301 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName B 2024-12-04T06:41:07,301 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:07,302 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:07,302 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,303 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:07,303 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName C 2024-12-04T06:41:07,303 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:07,304 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:07,304 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:07,304 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,304 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,305 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,305 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,306 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,306 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:41:07,307 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,310 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:41:07,310 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened ae9d50f9f78f5f45c22cc72ad2361274; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69042663, jitterRate=0.028815850615501404}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:41:07,310 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:07,311 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for ae9d50f9f78f5f45c22cc72ad2361274: Running coprocessor pre-open hook at 1733294467296Writing region info on filesystem at 1733294467296Initializing all the Stores at 1733294467297 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294467297Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294467297Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294467297Cleaning up temporary data from old regions at 1733294467306 (+9 ms)Running coprocessor post-open hooks at 1733294467310 (+4 ms)Region opened successfully at 1733294467311 (+1 ms) 2024-12-04T06:41:07,312 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., pid=34, masterSystemTime=1733294467291 2024-12-04T06:41:07,314 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:07,314 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:07,315 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:07,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:07,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-04T06:41:07,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 in 179 msec 2024-12-04T06:41:07,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T06:41:07,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, ASSIGN in 339 msec 2024-12-04T06:41:07,323 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:41:07,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294467323"}]},"ts":"1733294467323"} 2024-12-04T06:41:07,324 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:41:07,325 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:41:07,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 779 msec 2024-12-04T06:41:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T06:41:07,687 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:07,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511b0759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:07,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:07,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:07,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3722a31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:07,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:07,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:07,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:07,693 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:07,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:07,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:41:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:41:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-12-04T06:41:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:41:07,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:41:07,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:41:07,702 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:41:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:07,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741970_1146 (size=1097) 2024-12-04T06:41:08,120 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T06:41:08,120 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T06:41:08,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:41:08,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE}] 2024-12-04T06:41:08,131 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE 2024-12-04T06:41:08,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE because future has completed 2024-12-04T06:41:08,134 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:41:08,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:08,287 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:41:08,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing ae9d50f9f78f5f45c22cc72ad2361274, disabling compactions & flushes 2024-12-04T06:41:08,287 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. after waiting 0 ms 2024-12-04T06:41:08,287 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,291 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T06:41:08,292 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,292 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for ae9d50f9f78f5f45c22cc72ad2361274: Waiting for close lock at 1733294468287Running coprocessor pre-close hooks at 1733294468287Disabling compacts and flushes for region at 1733294468287Disabling writes for close at 1733294468287Writing region close event to WAL at 1733294468287Running coprocessor post-close hooks at 1733294468291 (+4 ms)Closed at 1733294468292 (+1 ms) 2024-12-04T06:41:08,292 WARN [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: ae9d50f9f78f5f45c22cc72ad2361274 to self. 2024-12-04T06:41:08,294 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,295 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=CLOSED 2024-12-04T06:41:08,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:08,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-04T06:41:08,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 in 163 msec 2024-12-04T06:41:08,300 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE; state=CLOSED, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=true 2024-12-04T06:41:08,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE because future has completed 2024-12-04T06:41:08,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:08,607 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,608 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:41:08,608 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,608 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:41:08,608 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,608 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,613 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,613 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:08,618 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName A 2024-12-04T06:41:08,620 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:08,620 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:08,621 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,621 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:08,622 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName B 2024-12-04T06:41:08,622 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:08,622 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:08,622 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,623 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:08,623 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae9d50f9f78f5f45c22cc72ad2361274 columnFamilyName C 2024-12-04T06:41:08,623 DEBUG [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:08,623 INFO [StoreOpener-ae9d50f9f78f5f45c22cc72ad2361274-1 {}] regionserver.HStore(327): Store=ae9d50f9f78f5f45c22cc72ad2361274/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:08,623 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,623 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,624 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,625 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,626 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:41:08,627 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,628 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened ae9d50f9f78f5f45c22cc72ad2361274; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59337271, jitterRate=-0.11580576002597809}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:41:08,628 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,630 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for ae9d50f9f78f5f45c22cc72ad2361274: Running coprocessor pre-open hook at 1733294468608Writing region info on filesystem at 1733294468608Initializing all the Stores at 1733294468609 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294468609Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294468609Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294468609Cleaning up temporary data from old regions at 1733294468626 (+17 ms)Running coprocessor post-open hooks at 1733294468628 (+2 ms)Region opened successfully at 1733294468630 (+2 ms) 2024-12-04T06:41:08,631 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., pid=39, masterSystemTime=1733294468604 2024-12-04T06:41:08,632 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,633 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,633 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=OPEN, openSeqNum=5, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:08,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-04T06:41:08,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 in 183 msec 2024-12-04T06:41:08,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-04T06:41:08,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, REOPEN/MOVE in 507 msec 2024-12-04T06:41:08,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-04T06:41:08,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-12-04T06:41:08,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-12-04T06:41:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-04T06:41:08,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:08,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc58118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,648 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f95389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1da99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37870970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@167a42dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454c5b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649c57a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79078822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8f38b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,660 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681041e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2cebc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbd945b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f22dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abba936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680235d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a6cf66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6547ab86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182dc420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ced825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4500591c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,672 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e853ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:08,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:08,677 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444285ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:08,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3177e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:08,679 DEBUG [Thread-717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,680 DEBUG [Thread-719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,681 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,682 DEBUG [Thread-721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-04T06:41:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:08,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:08,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:08,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:08,694 DEBUG [Thread-723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:08,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:08,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:08,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:08,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:08,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:08,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,700 DEBUG [Thread-730 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,709 DEBUG [Thread-732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,713 DEBUG [Thread-728 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294528717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294528718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1733294468702, max timestamp=1733294468713Num uniques 859; 2024-12-04T06:41:08,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1733294468702, max timestamp=1733294468713Num uniques 859; 2024-12-04T06:41:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-12-04T06:41:08,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-12-04T06:41:08,720 DEBUG [Thread-726 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=559, cellSize=25595, totalHeapSize=58376, min timestamp=1733294468702, max timestamp=1733294468713Num uniques 859; 2024-12-04T06:41:08,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=25595, suffix heap size=117064,new segment heap size=58376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=559 2024-12-04T06:41:08,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:41:08,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204451342bffab54ea1a5b24b3baf74683b_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294468691/Put/seqid=0 2024-12-04T06:41:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294528748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294528751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294528753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741971_1147 (size=12154) 2024-12-04T06:41:08,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:08,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204451342bffab54ea1a5b24b3baf74683b_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204451342bffab54ea1a5b24b3baf74683b_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:08,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f70f90827dc2417d94928c8868d56429, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:08,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f70f90827dc2417d94928c8868d56429 is 175, key is test_row_0/A:col10/1733294468691/Put/seqid=0 2024-12-04T06:41:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741972_1148 (size=30955) 2024-12-04T06:41:08,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f70f90827dc2417d94928c8868d56429 2024-12-04T06:41:08,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294528827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294528826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/7d284a1280b0435da88a629d71857a09 is 50, key is test_row_0/B:col10/1733294468691/Put/seqid=0 2024-12-04T06:41:08,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741973_1149 (size=12001) 2024-12-04T06:41:08,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:08,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:08,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294528868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294528868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294528868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:09,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294529036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294529047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294529076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294529076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294529077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/7d284a1280b0435da88a629d71857a09 2024-12-04T06:41:09,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4934cb46a7da4cf18f5fec12670eabef is 50, key is test_row_0/C:col10/1733294468691/Put/seqid=0 2024-12-04T06:41:09,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741974_1150 (size=12001) 2024-12-04T06:41:09,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4934cb46a7da4cf18f5fec12670eabef 2024-12-04T06:41:09,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f70f90827dc2417d94928c8868d56429 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429 2024-12-04T06:41:09,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429, entries=150, sequenceid=15, filesize=30.2 K 2024-12-04T06:41:09,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/7d284a1280b0435da88a629d71857a09 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09 2024-12-04T06:41:09,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:09,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T06:41:09,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4934cb46a7da4cf18f5fec12670eabef as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef 2024-12-04T06:41:09,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T06:41:09,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for ae9d50f9f78f5f45c22cc72ad2361274 in 642ms, sequenceid=15, compaction requested=false 2024-12-04T06:41:09,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:09,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=147.60 KB heapSize=387.33 KB 2024-12-04T06:41:09,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:09,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:09,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:09,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c46fdff4178a4c0c8681d6042c76c8a4_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294468726/Put/seqid=0 2024-12-04T06:41:09,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294529388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294529388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294529390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294529392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294529393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741975_1151 (size=14594) 2024-12-04T06:41:09,418 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c46fdff4178a4c0c8681d6042c76c8a4_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c46fdff4178a4c0c8681d6042c76c8a4_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:09,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/daf4a2127dd64c39be73fa7a42be3133, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:09,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/daf4a2127dd64c39be73fa7a42be3133 is 175, key is test_row_0/A:col10/1733294468726/Put/seqid=0 2024-12-04T06:41:09,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741976_1152 (size=39549) 2024-12-04T06:41:09,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/daf4a2127dd64c39be73fa7a42be3133 2024-12-04T06:41:09,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4454b7c9b23445b69d8cfa3b8f0684cf is 50, key is test_row_0/B:col10/1733294468726/Put/seqid=0 2024-12-04T06:41:09,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741977_1153 (size=12001) 2024-12-04T06:41:09,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4454b7c9b23445b69d8cfa3b8f0684cf 2024-12-04T06:41:09,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d53eea84d6f4c109f528f10e7606476 is 50, key is test_row_0/C:col10/1733294468726/Put/seqid=0 2024-12-04T06:41:09,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294529507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294529507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741978_1154 (size=12001) 2024-12-04T06:41:09,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294529716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294529717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294529896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294529906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294529907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:09,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:09,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:09,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:09,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d53eea84d6f4c109f528f10e7606476 2024-12-04T06:41:09,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/daf4a2127dd64c39be73fa7a42be3133 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133 2024-12-04T06:41:09,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133, entries=200, sequenceid=46, filesize=38.6 K 2024-12-04T06:41:09,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4454b7c9b23445b69d8cfa3b8f0684cf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf 2024-12-04T06:41:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T06:41:09,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d53eea84d6f4c109f528f10e7606476 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476 2024-12-04T06:41:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476, entries=150, sequenceid=46, filesize=11.7 K 2024-12-04T06:41:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.28 KB/396576, currentSize=46.96 KB/48090 for ae9d50f9f78f5f45c22cc72ad2361274 in 627ms, sequenceid=46, compaction requested=false 2024-12-04T06:41:09,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040fde76ca4a5a4a61a4e1ce9531c6c289_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741979_1155 (size=12154) 2024-12-04T06:41:10,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:10,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040fde76ca4a5a4a61a4e1ce9531c6c289_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fde76ca4a5a4a61a4e1ce9531c6c289_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:10,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f4b2a5f54e5847fcbb5cb7333661030a, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:10,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f4b2a5f54e5847fcbb5cb7333661030a is 175, key is test_row_0/A:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:10,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741980_1156 (size=30955) 2024-12-04T06:41:10,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294470096, max timestamp=1733294470132Num uniques 859; 2024-12-04T06:41:10,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294470096, max timestamp=1733294470132Num uniques 859; 2024-12-04T06:41:10,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:10,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:10,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294470096, max timestamp=1733294470132Num uniques 859; 2024-12-04T06:41:10,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:10,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294530193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294530195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294530306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294530307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f4b2a5f54e5847fcbb5cb7333661030a 2024-12-04T06:41:10,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1b0ee0494dbc4d2db1c276db6825397d is 50, key is test_row_0/B:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294530516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741981_1157 (size=12001) 2024-12-04T06:41:10,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294530517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294530827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294530827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:10,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:10,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1b0ee0494dbc4d2db1c276db6825397d 2024-12-04T06:41:10,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294530916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,922 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4d62004b8796405f9814defb42559eb5 is 50, key is test_row_0/C:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294530927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294530927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:10,935 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,937 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741982_1158 (size=12001) 2024-12-04T06:41:10,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4d62004b8796405f9814defb42559eb5 2024-12-04T06:41:10,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f4b2a5f54e5847fcbb5cb7333661030a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a 2024-12-04T06:41:10,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a, entries=150, sequenceid=57, filesize=30.2 K 2024-12-04T06:41:10,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1b0ee0494dbc4d2db1c276db6825397d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d 2024-12-04T06:41:10,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d, entries=150, sequenceid=57, filesize=11.7 K 2024-12-04T06:41:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4d62004b8796405f9814defb42559eb5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5 2024-12-04T06:41:10,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5, entries=150, sequenceid=57, filesize=11.7 K 2024-12-04T06:41:10,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae9d50f9f78f5f45c22cc72ad2361274 in 946ms, sequenceid=57, compaction requested=true 2024-12-04T06:41:10,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:10,983 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:10,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:10,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:10,984 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:10,985 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:10,985 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:10,985 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,985 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=99.1 K 2024-12-04T06:41:10,985 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,985 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a] 2024-12-04T06:41:10,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:10,986 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70f90827dc2417d94928c8868d56429, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294468691 2024-12-04T06:41:10,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:10,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:10,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:10,986 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf4a2127dd64c39be73fa7a42be3133, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733294468726 2024-12-04T06:41:10,987 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:10,987 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:10,987 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:10,987 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4b2a5f54e5847fcbb5cb7333661030a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:10,987 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.2 K 2024-12-04T06:41:10,988 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d284a1280b0435da88a629d71857a09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294468691 2024-12-04T06:41:10,989 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4454b7c9b23445b69d8cfa3b8f0684cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733294468726 2024-12-04T06:41:10,989 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b0ee0494dbc4d2db1c276db6825397d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:11,002 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:11,007 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:11,007 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/8d1d1eba1d5d476fb79a0023743a39f6 is 50, key is test_row_0/B:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-04T06:41:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:11,014 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a8272651b9d841b1804fad0ca3850696_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:11,014 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:11,022 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a8272651b9d841b1804fad0ca3850696_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:11,023 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a8272651b9d841b1804fad0ca3850696_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:11,031 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:41:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741983_1159 (size=12104) 2024-12-04T06:41:11,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040577706224b34d7db7ce4b936d12fce7_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294470193/Put/seqid=0 2024-12-04T06:41:11,058 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/8d1d1eba1d5d476fb79a0023743a39f6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/8d1d1eba1d5d476fb79a0023743a39f6 2024-12-04T06:41:11,068 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into 8d1d1eba1d5d476fb79a0023743a39f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:11,068 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:11,068 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294470984; duration=0sec 2024-12-04T06:41:11,068 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:11,068 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:11,069 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:11,071 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:11,072 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:11,072 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:11,073 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.2 K 2024-12-04T06:41:11,075 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4934cb46a7da4cf18f5fec12670eabef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294468691 2024-12-04T06:41:11,076 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d53eea84d6f4c109f528f10e7606476, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733294468726 2024-12-04T06:41:11,077 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d62004b8796405f9814defb42559eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:11,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741984_1160 (size=4469) 2024-12-04T06:41:11,105 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#140 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:11,106 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b79689c38dc64d069d6f535cb19a5fb2 is 175, key is test_row_0/A:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:11,124 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:11,124 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d2f6fa13db19452ab2766354f6316fb2 is 50, key is test_row_0/C:col10/1733294469391/Put/seqid=0 2024-12-04T06:41:11,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741985_1161 (size=12154) 2024-12-04T06:41:11,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741986_1162 (size=31058) 2024-12-04T06:41:11,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741987_1163 (size=12104) 2024-12-04T06:41:11,190 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d2f6fa13db19452ab2766354f6316fb2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d2f6fa13db19452ab2766354f6316fb2 2024-12-04T06:41:11,207 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into d2f6fa13db19452ab2766354f6316fb2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:11,207 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:11,216 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294470986; duration=0sec 2024-12-04T06:41:11,216 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:11,217 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:11,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:11,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294531382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294531384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294531487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294531496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:11,547 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412040577706224b34d7db7ce4b936d12fce7_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040577706224b34d7db7ce4b936d12fce7_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:11,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1c7bd97a48be4baab5eaf113626db1c2, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:11,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1c7bd97a48be4baab5eaf113626db1c2 is 175, key is test_row_0/A:col10/1733294470193/Put/seqid=0 2024-12-04T06:41:11,566 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b79689c38dc64d069d6f535cb19a5fb2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2 2024-12-04T06:41:11,573 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into b79689c38dc64d069d6f535cb19a5fb2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:11,573 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:11,573 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294470983; duration=0sec 2024-12-04T06:41:11,573 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:11,573 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:11,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741988_1164 (size=30955) 2024-12-04T06:41:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294531697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294531707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:11,987 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1c7bd97a48be4baab5eaf113626db1c2 2024-12-04T06:41:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4710eda8d912420e992d56784fee74bb is 50, key is test_row_0/B:col10/1733294470193/Put/seqid=0 2024-12-04T06:41:12,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741989_1165 (size=12001) 2024-12-04T06:41:12,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294532007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:12,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294532027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:12,403 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4710eda8d912420e992d56784fee74bb 2024-12-04T06:41:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/f4de3a98d3c345c69b99384483821bf1 is 50, key is test_row_0/C:col10/1733294470193/Put/seqid=0 2024-12-04T06:41:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741990_1166 (size=12001) 2024-12-04T06:41:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294532517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:12,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294532546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:12,817 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/f4de3a98d3c345c69b99384483821bf1 2024-12-04T06:41:12,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1c7bd97a48be4baab5eaf113626db1c2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2 2024-12-04T06:41:12,827 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2, entries=150, sequenceid=92, filesize=30.2 K 2024-12-04T06:41:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/4710eda8d912420e992d56784fee74bb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb 2024-12-04T06:41:12,833 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb, entries=150, sequenceid=92, filesize=11.7 K 2024-12-04T06:41:12,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/f4de3a98d3c345c69b99384483821bf1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1 2024-12-04T06:41:12,838 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1, entries=150, sequenceid=92, filesize=11.7 K 2024-12-04T06:41:12,839 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for ae9d50f9f78f5f45c22cc72ad2361274 in 1825ms, sequenceid=92, compaction requested=false 2024-12-04T06:41:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:12,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:12,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-04T06:41:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-04T06:41:12,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-04T06:41:12,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1500 sec 2024-12-04T06:41:12,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 4.1590 sec 2024-12-04T06:41:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-04T06:41:12,857 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:12,858 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-04T06:41:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:12,860 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:12,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:12,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:12,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:41:12,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:12,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204014061710150461cbe96e4413d9b76e8_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:12,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741991_1167 (size=12154) 2024-12-04T06:41:12,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:12,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204014061710150461cbe96e4413d9b76e8_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204014061710150461cbe96e4413d9b76e8_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:12,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/462e6223d7154f8681ca2b1c6e290cdd, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:13,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/462e6223d7154f8681ca2b1c6e290cdd is 175, key is test_row_0/A:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:13,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741992_1168 (size=30955) 2024-12-04T06:41:13,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/462e6223d7154f8681ca2b1c6e290cdd 2024-12-04T06:41:13,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294533001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/93d82afa74a649978673cb56d9d7db91 is 50, key is test_row_0/B:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:13,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741993_1169 (size=12001) 2024-12-04T06:41:13,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294533005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294533005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294533117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294533127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294533128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:13,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294533337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294533338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294533338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/93d82afa74a649978673cb56d9d7db91 2024-12-04T06:41:13,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/68e0a57e1c75400ab2f21e351f383bcd is 50, key is test_row_0/C:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:13,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741994_1170 (size=12001) 2024-12-04T06:41:13,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294533536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,538 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294533567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,571 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294533646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294533647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294533657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:13,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:13,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:13,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:13,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/68e0a57e1c75400ab2f21e351f383bcd 2024-12-04T06:41:13,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/462e6223d7154f8681ca2b1c6e290cdd as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd 2024-12-04T06:41:13,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd, entries=150, sequenceid=108, filesize=30.2 K 2024-12-04T06:41:13,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/93d82afa74a649978673cb56d9d7db91 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91 2024-12-04T06:41:13,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T06:41:13,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/68e0a57e1c75400ab2f21e351f383bcd as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd 2024-12-04T06:41:13,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd, entries=150, sequenceid=108, filesize=11.7 K 2024-12-04T06:41:13,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ae9d50f9f78f5f45c22cc72ad2361274 in 914ms, sequenceid=108, compaction requested=true 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:13,864 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:13,864 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:13,865 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:13,865 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:13,865 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,865 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:13,865 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=90.8 K 2024-12-04T06:41:13,866 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:13,866 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,866 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd] 2024-12-04T06:41:13,866 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,866 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/8d1d1eba1d5d476fb79a0023743a39f6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.3 K 2024-12-04T06:41:13,866 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79689c38dc64d069d6f535cb19a5fb2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:13,866 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1d1eba1d5d476fb79a0023743a39f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:13,867 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c7bd97a48be4baab5eaf113626db1c2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294470185 2024-12-04T06:41:13,867 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462e6223d7154f8681ca2b1c6e290cdd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:13,868 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4710eda8d912420e992d56784fee74bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294470185 2024-12-04T06:41:13,869 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 93d82afa74a649978673cb56d9d7db91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:13,877 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:13,879 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:13,880 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/f40849c3d9cb46c2997a8e0210f12232 is 50, key is test_row_0/B:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:13,881 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120442cb2fa56a6444d1aed9fecf8a89205a_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:13,883 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120442cb2fa56a6444d1aed9fecf8a89205a_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:13,884 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120442cb2fa56a6444d1aed9fecf8a89205a_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:13,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741995_1171 (size=12207) 2024-12-04T06:41:13,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741996_1172 (size=4469) 2024-12-04T06:41:13,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-04T06:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:13,936 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:13,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:13,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120487d57eb661ff4ab48b1d7acd3e3c8e4b_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294472989/Put/seqid=0 2024-12-04T06:41:13,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741997_1173 (size=12154) 2024-12-04T06:41:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:14,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294534173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294534174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294534181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294534287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294534287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,295 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/f40849c3d9cb46c2997a8e0210f12232 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f40849c3d9cb46c2997a8e0210f12232 2024-12-04T06:41:14,295 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#149 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:14,296 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/dceda970b6d8477fa74e0bb42bc8b16c is 175, key is test_row_0/A:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:14,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294534297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,303 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into f40849c3d9cb46c2997a8e0210f12232(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:14,303 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:14,303 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294473864; duration=0sec 2024-12-04T06:41:14,303 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:14,303 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:14,303 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:14,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741998_1174 (size=31161) 2024-12-04T06:41:14,305 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:14,305 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:14,305 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:14,305 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d2f6fa13db19452ab2766354f6316fb2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.3 K 2024-12-04T06:41:14,306 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f6fa13db19452ab2766354f6316fb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733294469389 2024-12-04T06:41:14,307 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f4de3a98d3c345c69b99384483821bf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294470185 2024-12-04T06:41:14,307 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e0a57e1c75400ab2f21e351f383bcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:14,315 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:14,316 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d688f85d13d247fbbed1b3ac6f861e97 is 50, key is test_row_0/C:col10/1733294471352/Put/seqid=0 2024-12-04T06:41:14,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741999_1175 (size=12207) 2024-12-04T06:41:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:14,358 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120487d57eb661ff4ab48b1d7acd3e3c8e4b_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487d57eb661ff4ab48b1d7acd3e3c8e4b_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:14,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/8289ae78d40b427f95b4b8722575a3cc, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/8289ae78d40b427f95b4b8722575a3cc is 175, key is test_row_0/A:col10/1733294472989/Put/seqid=0 2024-12-04T06:41:14,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742000_1176 (size=30955) 2024-12-04T06:41:14,373 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/8289ae78d40b427f95b4b8722575a3cc 2024-12-04T06:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/bcdd57c8f8624f3a943c8a6f35b6d726 is 50, key is test_row_0/B:col10/1733294472989/Put/seqid=0 2024-12-04T06:41:14,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742001_1177 (size=12001) 2024-12-04T06:41:14,441 INFO [master/607fd5c6574c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T06:41:14,441 INFO [master/607fd5c6574c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T06:41:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294534496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294534496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294534506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,710 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/dceda970b6d8477fa74e0bb42bc8b16c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c 2024-12-04T06:41:14,716 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into dceda970b6d8477fa74e0bb42bc8b16c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:14,716 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:14,716 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294473864; duration=0sec 2024-12-04T06:41:14,716 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:14,716 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:14,725 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d688f85d13d247fbbed1b3ac6f861e97 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d688f85d13d247fbbed1b3ac6f861e97 2024-12-04T06:41:14,731 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into d688f85d13d247fbbed1b3ac6f861e97(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:14,731 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:14,731 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294473864; duration=0sec 2024-12-04T06:41:14,732 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:14,732 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:14,788 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/bcdd57c8f8624f3a943c8a6f35b6d726 2024-12-04T06:41:14,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/962f1a62c3df4e5f9bf77e2240fe986d is 50, key is test_row_0/C:col10/1733294472989/Put/seqid=0 2024-12-04T06:41:14,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742002_1178 (size=12001) 2024-12-04T06:41:14,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294534807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294534807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:14,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294534816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:15,201 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/962f1a62c3df4e5f9bf77e2240fe986d 2024-12-04T06:41:15,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/8289ae78d40b427f95b4b8722575a3cc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc 2024-12-04T06:41:15,213 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc, entries=150, sequenceid=127, filesize=30.2 K 2024-12-04T06:41:15,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/bcdd57c8f8624f3a943c8a6f35b6d726 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726 2024-12-04T06:41:15,219 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726, entries=150, sequenceid=127, filesize=11.7 K 2024-12-04T06:41:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/962f1a62c3df4e5f9bf77e2240fe986d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d 2024-12-04T06:41:15,224 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d, entries=150, sequenceid=127, filesize=11.7 K 2024-12-04T06:41:15,225 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ae9d50f9f78f5f45c22cc72ad2361274 in 1289ms, sequenceid=127, compaction requested=false 2024-12-04T06:41:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:15,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-04T06:41:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-04T06:41:15,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T06:41:15,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3660 sec 2024-12-04T06:41:15,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.3720 sec 2024-12-04T06:41:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:15,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:15,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043ffc5a9897dc43d1b62fe9abd2bc39d1_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:15,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742003_1179 (size=17284) 2024-12-04T06:41:15,348 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:15,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043ffc5a9897dc43d1b62fe9abd2bc39d1_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043ffc5a9897dc43d1b62fe9abd2bc39d1_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:15,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/d8e1264551464d538299c4ccb7f1419a, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:15,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/d8e1264551464d538299c4ccb7f1419a is 175, key is test_row_0/A:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294535351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742004_1180 (size=48389) 2024-12-04T06:41:15,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294535358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294535358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/d8e1264551464d538299c4ccb7f1419a 2024-12-04T06:41:15,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/28e105d74eeb427fa1c24de588368b13 is 50, key is test_row_0/B:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:15,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742005_1181 (size=12151) 2024-12-04T06:41:15,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294535467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294535467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294535467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294535557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,560 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294535587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,589 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294535677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294535677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294535677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/28e105d74eeb427fa1c24de588368b13 2024-12-04T06:41:15,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/951d2ba6c6454ef1a0ddcf6a99de3c4a is 50, key is test_row_0/C:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:15,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742006_1182 (size=12151) 2024-12-04T06:41:15,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294535986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294535987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294535987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/951d2ba6c6454ef1a0ddcf6a99de3c4a 2024-12-04T06:41:16,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/d8e1264551464d538299c4ccb7f1419a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a 2024-12-04T06:41:16,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a, entries=250, sequenceid=145, filesize=47.3 K 2024-12-04T06:41:16,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/28e105d74eeb427fa1c24de588368b13 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13 2024-12-04T06:41:16,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13, entries=150, sequenceid=145, filesize=11.9 K 2024-12-04T06:41:16,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/951d2ba6c6454ef1a0ddcf6a99de3c4a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a 2024-12-04T06:41:16,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a, entries=150, sequenceid=145, filesize=11.9 K 2024-12-04T06:41:16,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for ae9d50f9f78f5f45c22cc72ad2361274 in 902ms, sequenceid=145, compaction requested=true 2024-12-04T06:41:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:16,223 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:16,224 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:16,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:16,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:16,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:16,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:16,225 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:16,225 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:16,225 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:16,225 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:16,225 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:16,225 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:16,225 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f40849c3d9cb46c2997a8e0210f12232, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.5 K 2024-12-04T06:41:16,225 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=107.9 K 2024-12-04T06:41:16,225 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:16,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a] 2024-12-04T06:41:16,226 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f40849c3d9cb46c2997a8e0210f12232, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:16,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting dceda970b6d8477fa74e0bb42bc8b16c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:16,227 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8289ae78d40b427f95b4b8722575a3cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294472989 2024-12-04T06:41:16,227 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bcdd57c8f8624f3a943c8a6f35b6d726, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294472989 2024-12-04T06:41:16,227 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 28e105d74eeb427fa1c24de588368b13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:16,227 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e1264551464d538299c4ccb7f1419a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:16,237 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:16,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/fe078e2b62ef4cf29044a86bf088fb74 is 50, key is test_row_0/B:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:16,240 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:16,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120474c9dad82b3d405d8a1ff11136de5868_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:16,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742007_1183 (size=12459) 2024-12-04T06:41:16,250 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120474c9dad82b3d405d8a1ff11136de5868_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:16,250 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120474c9dad82b3d405d8a1ff11136de5868_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:16,261 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/fe078e2b62ef4cf29044a86bf088fb74 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/fe078e2b62ef4cf29044a86bf088fb74 2024-12-04T06:41:16,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742008_1184 (size=4469) 2024-12-04T06:41:16,267 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into fe078e2b62ef4cf29044a86bf088fb74(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:16,267 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:16,267 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294476223; duration=0sec 2024-12-04T06:41:16,268 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:16,268 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:16,268 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:16,269 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#159 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:16,270 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b04d42fe45094f23b70f2fa582e5dff2 is 175, key is test_row_0/A:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:16,273 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:16,273 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:16,273 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:16,273 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d688f85d13d247fbbed1b3ac6f861e97, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=35.5 K 2024-12-04T06:41:16,274 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d688f85d13d247fbbed1b3ac6f861e97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733294471352 2024-12-04T06:41:16,274 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 962f1a62c3df4e5f9bf77e2240fe986d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294472989 2024-12-04T06:41:16,274 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 951d2ba6c6454ef1a0ddcf6a99de3c4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:16,291 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:16,291 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/2aa45fb43c564e09b134b99e9f4f4bb9 is 50, key is test_row_0/C:col10/1733294474179/Put/seqid=0 2024-12-04T06:41:16,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742009_1185 (size=31413) 2024-12-04T06:41:16,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742010_1186 (size=12459) 2024-12-04T06:41:16,307 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b04d42fe45094f23b70f2fa582e5dff2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2 2024-12-04T06:41:16,313 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into b04d42fe45094f23b70f2fa582e5dff2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:16,313 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:16,314 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294476223; duration=0sec 2024-12-04T06:41:16,314 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:16,314 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:16,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:41:16,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:16,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:16,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204552c029e141e41b2a13f830b657a51c2_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294476497/Put/seqid=0 2024-12-04T06:41:16,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742011_1187 (size=17284) 2024-12-04T06:41:16,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294536510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294536511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294536511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294536627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294536627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294536627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,708 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/2aa45fb43c564e09b134b99e9f4f4bb9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/2aa45fb43c564e09b134b99e9f4f4bb9 2024-12-04T06:41:16,715 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 2aa45fb43c564e09b134b99e9f4f4bb9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:16,715 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:16,716 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294476224; duration=0sec 2024-12-04T06:41:16,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:16,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294536837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294536837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294536837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:16,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:16,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:16,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204552c029e141e41b2a13f830b657a51c2_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204552c029e141e41b2a13f830b657a51c2_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:16,921 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c9b417f32018436097512af86ee8dfdf, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:16,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c9b417f32018436097512af86ee8dfdf is 175, key is test_row_0/A:col10/1733294476497/Put/seqid=0 2024-12-04T06:41:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742012_1188 (size=48389) 2024-12-04T06:41:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T06:41:17,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:17,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-04T06:41:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:17,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:17,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:17,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:17,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294537146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294537147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294537147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T06:41:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,329 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c9b417f32018436097512af86ee8dfdf 2024-12-04T06:41:17,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T06:41:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/0b5581c8f6e9430e8cc4bde0831c8e85 is 50, key is test_row_0/B:col10/1733294476497/Put/seqid=0 2024-12-04T06:41:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:17,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742013_1189 (size=12151) 2024-12-04T06:41:17,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/0b5581c8f6e9430e8cc4bde0831c8e85 2024-12-04T06:41:17,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8bed6e7fde224b18bfa7eeb93382417e is 50, key is test_row_0/C:col10/1733294476497/Put/seqid=0 2024-12-04T06:41:17,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742014_1190 (size=12151) 2024-12-04T06:41:17,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T06:41:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T06:41:17,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:17,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:17,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294537657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294537657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294537657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:17,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8bed6e7fde224b18bfa7eeb93382417e 2024-12-04T06:41:17,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c9b417f32018436097512af86ee8dfdf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf 2024-12-04T06:41:17,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf, entries=250, sequenceid=167, filesize=47.3 K 2024-12-04T06:41:17,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/0b5581c8f6e9430e8cc4bde0831c8e85 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85 2024-12-04T06:41:17,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85, entries=150, sequenceid=167, filesize=11.9 K 2024-12-04T06:41:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8bed6e7fde224b18bfa7eeb93382417e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e 2024-12-04T06:41:17,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e, entries=150, sequenceid=167, filesize=11.9 K 2024-12-04T06:41:17,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae9d50f9f78f5f45c22cc72ad2361274 in 1285ms, sequenceid=167, compaction requested=false 2024-12-04T06:41:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:17,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-04T06:41:17,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:17,797 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:17,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204db570c6273fe44a6936a67eb1a0cc540_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294476509/Put/seqid=0 2024-12-04T06:41:17,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742015_1191 (size=12304) 2024-12-04T06:41:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,214 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204db570c6273fe44a6936a67eb1a0cc540_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204db570c6273fe44a6936a67eb1a0cc540_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8bff16f840940abadad8b820bff7bf2, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:18,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8bff16f840940abadad8b820bff7bf2 is 175, key is test_row_0/A:col10/1733294476509/Put/seqid=0 2024-12-04T06:41:18,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742016_1192 (size=31105) 2024-12-04T06:41:18,221 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8bff16f840940abadad8b820bff7bf2 2024-12-04T06:41:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa is 50, key is test_row_0/B:col10/1733294476509/Put/seqid=0 2024-12-04T06:41:18,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742017_1193 (size=12151) 2024-12-04T06:41:18,235 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa 2024-12-04T06:41:18,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4904892f6038487783b0b57e2f8cb538 is 50, key is test_row_0/C:col10/1733294476509/Put/seqid=0 2024-12-04T06:41:18,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742018_1194 (size=12151) 2024-12-04T06:41:18,647 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4904892f6038487783b0b57e2f8cb538 2024-12-04T06:41:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8bff16f840940abadad8b820bff7bf2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2 2024-12-04T06:41:18,658 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2, entries=150, sequenceid=180, filesize=30.4 K 2024-12-04T06:41:18,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa 2024-12-04T06:41:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,663 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa, entries=150, sequenceid=180, filesize=11.9 K 2024-12-04T06:41:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4904892f6038487783b0b57e2f8cb538 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538 2024-12-04T06:41:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,669 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538, entries=150, sequenceid=180, filesize=11.9 K 2024-12-04T06:41:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,670 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ae9d50f9f78f5f45c22cc72ad2361274 in 873ms, sequenceid=180, compaction requested=true 2024-12-04T06:41:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-04T06:41:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-04T06:41:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T06:41:18,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-04T06:41:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6490 sec 2024-12-04T06:41:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ede2e8ee900242e3a20f8de656d48c3f_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742019_1195 (size=22268) 2024-12-04T06:41:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ede2e8ee900242e3a20f8de656d48c3f_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ede2e8ee900242e3a20f8de656d48c3f_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8b58802b54148c3b4a91df4c6670ec0, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8b58802b54148c3b4a91df4c6670ec0 is 175, key is test_row_0/A:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742020_1196 (size=65669) 2024-12-04T06:41:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1733294478799, max timestamp=1733294478858Num uniques 859; 2024-12-04T06:41:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1733294478799, max timestamp=1733294478858Num uniques 859; 2024-12-04T06:41:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-12-04T06:41:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-12-04T06:41:18,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1733294478799, max timestamp=1733294478858Num uniques 859; 2024-12-04T06:41:18,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-12-04T06:41:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:18,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294538883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294538893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294538893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:18,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294538996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294539007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294539007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-04T06:41:19,177 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:19,177 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-04T06:41:19,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:19,181 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:19,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:19,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8b58802b54148c3b4a91df4c6670ec0 2024-12-04T06:41:19,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294539207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b4622628a82448f8aaf23f82a7941201 is 50, key is test_row_0/B:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742021_1197 (size=9757) 2024-12-04T06:41:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294539216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294539216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:19,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294539517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294539527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294539527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294539576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,580 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294539607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:19,611 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:19,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:19,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b4622628a82448f8aaf23f82a7941201 2024-12-04T06:41:19,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/55c01f03b7ed47739bfe7171c9113503 is 50, key is test_row_0/C:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:19,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742022_1198 (size=9757) 2024-12-04T06:41:19,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:19,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:19,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294540026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:20,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/55c01f03b7ed47739bfe7171c9113503 2024-12-04T06:41:20,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/a8b58802b54148c3b4a91df4c6670ec0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0 2024-12-04T06:41:20,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0, entries=350, sequenceid=190, filesize=64.1 K 2024-12-04T06:41:20,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b4622628a82448f8aaf23f82a7941201 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201 2024-12-04T06:41:20,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294540047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294540047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:20,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201, entries=100, sequenceid=190, filesize=9.5 K 2024-12-04T06:41:20,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/55c01f03b7ed47739bfe7171c9113503 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503 2024-12-04T06:41:20,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503, entries=100, sequenceid=190, filesize=9.5 K 2024-12-04T06:41:20,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ae9d50f9f78f5f45c22cc72ad2361274 in 1319ms, sequenceid=190, compaction requested=true 2024-12-04T06:41:20,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:20,073 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:20,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:20,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:20,074 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:20,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:20,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:20,076 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:20,076 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:20,076 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,076 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176576 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:20,076 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:20,076 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,077 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=172.4 K 2024-12-04T06:41:20,077 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,077 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0] 2024-12-04T06:41:20,077 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/fe078e2b62ef4cf29044a86bf088fb74, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=45.4 K 2024-12-04T06:41:20,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:20,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:20,078 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fe078e2b62ef4cf29044a86bf088fb74, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:20,078 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04d42fe45094f23b70f2fa582e5dff2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:20,079 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5581c8f6e9430e8cc4bde0831c8e85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733294475325 2024-12-04T06:41:20,079 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b417f32018436097512af86ee8dfdf, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733294475325 2024-12-04T06:41:20,080 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a2ef9b2cd51d4de1a3eaceeb5c68fcaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733294476508 2024-12-04T06:41:20,080 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8bff16f840940abadad8b820bff7bf2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733294476508 2024-12-04T06:41:20,080 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b4622628a82448f8aaf23f82a7941201, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294478746 2024-12-04T06:41:20,081 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b58802b54148c3b4a91df4c6670ec0, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294478700 2024-12-04T06:41:20,101 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:20,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-04T06:41:20,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,105 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=147.60 KB heapSize=382.64 KB 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:20,120 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#171 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:20,121 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/46a9cb50d0cb4785b193eb53d26d4d4c is 50, key is test_row_0/B:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:20,138 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204a50b55adf6ac4539a6be07eb3b6a435a_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:20,141 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204a50b55adf6ac4539a6be07eb3b6a435a_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:20,141 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a50b55adf6ac4539a6be07eb3b6a435a_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:20,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120468df178b6803491eaaa3eae1957b62be_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294478891/Put/seqid=0 2024-12-04T06:41:20,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742023_1199 (size=12595) 2024-12-04T06:41:20,227 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/46a9cb50d0cb4785b193eb53d26d4d4c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/46a9cb50d0cb4785b193eb53d26d4d4c 2024-12-04T06:41:20,233 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into 46a9cb50d0cb4785b193eb53d26d4d4c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:20,234 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:20,234 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=12, startTime=1733294480074; duration=0sec 2024-12-04T06:41:20,234 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:20,234 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:20,234 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:20,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:20,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:20,236 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,236 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/2aa45fb43c564e09b134b99e9f4f4bb9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=45.4 K 2024-12-04T06:41:20,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa45fb43c564e09b134b99e9f4f4bb9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733294474166 2024-12-04T06:41:20,237 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bed6e7fde224b18bfa7eeb93382417e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733294475325 2024-12-04T06:41:20,237 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4904892f6038487783b0b57e2f8cb538, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733294476508 2024-12-04T06:41:20,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c01f03b7ed47739bfe7171c9113503, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294478746 2024-12-04T06:41:20,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742024_1200 (size=4469) 2024-12-04T06:41:20,248 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#170 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:20,248 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/bd0dc3f151ec40cba8c2b981315d3f4d is 175, key is test_row_0/A:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:20,267 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:20,267 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7c15c202ace442798ca4f5fdb575ea07 is 50, key is test_row_0/C:col10/1733294478746/Put/seqid=0 2024-12-04T06:41:20,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742025_1201 (size=12304) 2024-12-04T06:41:20,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742026_1202 (size=31656) 2024-12-04T06:41:20,300 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/bd0dc3f151ec40cba8c2b981315d3f4d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d 2024-12-04T06:41:20,308 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into bd0dc3f151ec40cba8c2b981315d3f4d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:20,308 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:20,309 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=12, startTime=1733294480073; duration=0sec 2024-12-04T06:41:20,309 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:20,309 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:20,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742027_1203 (size=12595) 2024-12-04T06:41:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,698 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120468df178b6803491eaaa3eae1957b62be_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120468df178b6803491eaaa3eae1957b62be_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c78a3725dbab45cb90cf9cc781186efb, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c78a3725dbab45cb90cf9cc781186efb is 175, key is test_row_0/A:col10/1733294478891/Put/seqid=0 2024-12-04T06:41:20,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742028_1204 (size=31105) 2024-12-04T06:41:20,737 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c78a3725dbab45cb90cf9cc781186efb 2024-12-04T06:41:20,753 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7c15c202ace442798ca4f5fdb575ea07 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7c15c202ace442798ca4f5fdb575ea07 2024-12-04T06:41:20,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/d6df83896e794b8fbe1ba47ff232cf52 is 50, key is test_row_0/B:col10/1733294478891/Put/seqid=0 2024-12-04T06:41:20,765 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 7c15c202ace442798ca4f5fdb575ea07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:20,765 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:20,765 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=12, startTime=1733294480077; duration=0sec 2024-12-04T06:41:20,765 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:20,766 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:20,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742029_1205 (size=12151) 2024-12-04T06:41:20,799 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/d6df83896e794b8fbe1ba47ff232cf52 2024-12-04T06:41:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8a971fa0f28a44d3a8a61e2c629a4ffd is 50, key is test_row_0/C:col10/1733294478891/Put/seqid=0 2024-12-04T06:41:20,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742030_1206 (size=12151) 2024-12-04T06:41:20,880 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8a971fa0f28a44d3a8a61e2c629a4ffd 2024-12-04T06:41:20,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/c78a3725dbab45cb90cf9cc781186efb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb 2024-12-04T06:41:20,891 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb, entries=150, sequenceid=220, filesize=30.4 K 2024-12-04T06:41:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/d6df83896e794b8fbe1ba47ff232cf52 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52 2024-12-04T06:41:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,899 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T06:41:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/8a971fa0f28a44d3a8a61e2c629a4ffd as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd 2024-12-04T06:41:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,906 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd, entries=150, sequenceid=220, filesize=11.9 K 2024-12-04T06:41:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,907 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~382.59 KB/391776, currentSize=0 B/0 for ae9d50f9f78f5f45c22cc72ad2361274 in 802ms, sequenceid=220, compaction requested=false 2024-12-04T06:41:20,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:20,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-04T06:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-04T06:41:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-04T06:41:20,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-12-04T06:41:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7360 sec 2024-12-04T06:41:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:21,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:21,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042cfbc9f7390e4823bdaf638c2ac38fa3_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:21,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742031_1207 (size=14794) 2024-12-04T06:41:21,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:21,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412042cfbc9f7390e4823bdaf638c2ac38fa3_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042cfbc9f7390e4823bdaf638c2ac38fa3_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:21,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1ea7dcf0256c461697e8f4a509ef399e, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:21,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1ea7dcf0256c461697e8f4a509ef399e is 175, key is test_row_0/A:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:21,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294481164, max timestamp=1733294481209Num uniques 859; 2024-12-04T06:41:21,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:41:21,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294481183, max timestamp=1733294481209Num uniques 859; 2024-12-04T06:41:21,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:21,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294481183, max timestamp=1733294481209Num uniques 859; 2024-12-04T06:41:21,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:21,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742032_1208 (size=39749) 2024-12-04T06:41:21,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294541289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294541300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294541301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-04T06:41:21,327 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:21,328 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-04T06:41:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:21,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:21,331 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:21,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:21,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294541637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294541637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294541637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:21,652 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1ea7dcf0256c461697e8f4a509ef399e 2024-12-04T06:41:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/f8584a7ec47d4267b491a7266309b2b4 is 50, key is test_row_0/B:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:21,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742033_1209 (size=12151) 2024-12-04T06:41:21,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/f8584a7ec47d4267b491a7266309b2b4 2024-12-04T06:41:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d5b184cecdcc471393f680c72bee8e44 is 50, key is test_row_0/C:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:21,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742034_1210 (size=12151) 2024-12-04T06:41:21,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:21,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294541946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294541957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294541966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:22,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:22,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d5b184cecdcc471393f680c72bee8e44 2024-12-04T06:41:22,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1ea7dcf0256c461697e8f4a509ef399e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e 2024-12-04T06:41:22,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e, entries=200, sequenceid=234, filesize=38.8 K 2024-12-04T06:41:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/f8584a7ec47d4267b491a7266309b2b4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4 2024-12-04T06:41:22,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4, entries=150, sequenceid=234, filesize=11.9 K 2024-12-04T06:41:22,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d5b184cecdcc471393f680c72bee8e44 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44 2024-12-04T06:41:22,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44, entries=150, sequenceid=234, filesize=11.9 K 2024-12-04T06:41:22,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for ae9d50f9f78f5f45c22cc72ad2361274 in 1067ms, sequenceid=234, compaction requested=true 2024-12-04T06:41:22,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:22,209 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:22,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:22,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:22,209 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:22,210 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102510 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:22,210 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:22,210 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,211 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=100.1 K 2024-12-04T06:41:22,211 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e] 2024-12-04T06:41:22,211 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:22,211 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:22,211 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,211 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/46a9cb50d0cb4785b193eb53d26d4d4c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.0 K 2024-12-04T06:41:22,212 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0dc3f151ec40cba8c2b981315d3f4d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294476509 2024-12-04T06:41:22,212 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a9cb50d0cb4785b193eb53d26d4d4c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294476509 2024-12-04T06:41:22,212 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c78a3725dbab45cb90cf9cc781186efb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733294478878 2024-12-04T06:41:22,213 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d6df83896e794b8fbe1ba47ff232cf52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733294478878 2024-12-04T06:41:22,213 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea7dcf0256c461697e8f4a509ef399e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:22,213 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f8584a7ec47d4267b491a7266309b2b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:22,227 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:22,239 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:22,239 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/6770aab06c844fecaa5cc84097440f6d is 50, key is test_row_0/B:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:22,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204c620272da6e44b10974c71105f279a07_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:22,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204c620272da6e44b10974c71105f279a07_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:22,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c620272da6e44b10974c71105f279a07_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:22,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742035_1211 (size=4469) 2024-12-04T06:41:22,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-04T06:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,254 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:22,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742036_1212 (size=12697) 2024-12-04T06:41:22,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049cc0bf70dbee475ebeb5a138eae9bd36_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294481281/Put/seqid=0 2024-12-04T06:41:22,269 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/6770aab06c844fecaa5cc84097440f6d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/6770aab06c844fecaa5cc84097440f6d 2024-12-04T06:41:22,274 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into 6770aab06c844fecaa5cc84097440f6d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:22,275 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:22,275 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294482209; duration=0sec 2024-12-04T06:41:22,275 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:22,275 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:22,275 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:22,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:22,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:22,276 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:22,276 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7c15c202ace442798ca4f5fdb575ea07, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.0 K 2024-12-04T06:41:22,278 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c15c202ace442798ca4f5fdb575ea07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294476509 2024-12-04T06:41:22,278 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a971fa0f28a44d3a8a61e2c629a4ffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733294478878 2024-12-04T06:41:22,279 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b184cecdcc471393f680c72bee8e44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742037_1213 (size=12454) 2024-12-04T06:41:22,315 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:22,316 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1a524dc312b1484683b06e74fd0e9381 is 50, key is test_row_0/C:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742038_1214 (size=12697) 2024-12-04T06:41:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:22,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294542483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294542491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294542492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294542597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294542617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294542619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,651 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:22,652 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1da5405d36f742528bd538893750e697 is 175, key is test_row_0/A:col10/1733294481141/Put/seqid=0 2024-12-04T06:41:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742039_1215 (size=31651) 2024-12-04T06:41:22,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:22,686 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049cc0bf70dbee475ebeb5a138eae9bd36_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049cc0bf70dbee475ebeb5a138eae9bd36_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f6d4071cc7c44e1d8a41fab304056dcf, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f6d4071cc7c44e1d8a41fab304056dcf is 175, key is test_row_0/A:col10/1733294481281/Put/seqid=0 2024-12-04T06:41:22,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742040_1216 (size=31255) 2024-12-04T06:41:22,696 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f6d4071cc7c44e1d8a41fab304056dcf 2024-12-04T06:41:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1a9cad6ae8b443128e0e8e6347f9dac0 is 50, key is test_row_0/B:col10/1733294481281/Put/seqid=0 2024-12-04T06:41:22,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742041_1217 (size=12301) 2024-12-04T06:41:22,765 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1a524dc312b1484683b06e74fd0e9381 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1a524dc312b1484683b06e74fd0e9381 2024-12-04T06:41:22,777 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 1a524dc312b1484683b06e74fd0e9381(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:22,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:22,777 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294482210; duration=0sec 2024-12-04T06:41:22,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:22,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:22,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294542807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294542836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294542836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,063 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/1da5405d36f742528bd538893750e697 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697 2024-12-04T06:41:23,067 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into 1da5405d36f742528bd538893750e697(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:23,067 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:23,067 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294482209; duration=0sec 2024-12-04T06:41:23,068 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:23,068 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:23,110 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1a9cad6ae8b443128e0e8e6347f9dac0 2024-12-04T06:41:23,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/b7feaccffca34d9cb833bfac9486dd62 is 50, key is test_row_0/C:col10/1733294481281/Put/seqid=0 2024-12-04T06:41:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294543117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742042_1218 (size=12301) 2024-12-04T06:41:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294543148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294543149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:23,540 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/b7feaccffca34d9cb833bfac9486dd62 2024-12-04T06:41:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f6d4071cc7c44e1d8a41fab304056dcf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf 2024-12-04T06:41:23,553 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf, entries=150, sequenceid=269, filesize=30.5 K 2024-12-04T06:41:23,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/1a9cad6ae8b443128e0e8e6347f9dac0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0 2024-12-04T06:41:23,564 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0, entries=150, sequenceid=269, filesize=12.0 K 2024-12-04T06:41:23,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/b7feaccffca34d9cb833bfac9486dd62 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62 2024-12-04T06:41:23,570 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62, entries=150, sequenceid=269, filesize=12.0 K 2024-12-04T06:41:23,571 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for ae9d50f9f78f5f45c22cc72ad2361274 in 1317ms, sequenceid=269, compaction requested=false 2024-12-04T06:41:23,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:23,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:23,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-04T06:41:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-04T06:41:23,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-04T06:41:23,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-12-04T06:41:23,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2530 sec 2024-12-04T06:41:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:23,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:41:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:23,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:23,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:23,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:23,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044c67bf39c8084da48d881628eeed2467_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:23,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742043_1219 (size=14994) 2024-12-04T06:41:23,704 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412044c67bf39c8084da48d881628eeed2467_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c67bf39c8084da48d881628eeed2467_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:23,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e71625f4e3b541489a4090dfd7a56995, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:23,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e71625f4e3b541489a4090dfd7a56995 is 175, key is test_row_0/A:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294543715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294543717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294543720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742044_1220 (size=39949) 2024-12-04T06:41:23,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e71625f4e3b541489a4090dfd7a56995 2024-12-04T06:41:23,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/24a571cfc52c4a4db001e8a946d98564 is 50, key is test_row_0/B:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:23,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742045_1221 (size=12301) 2024-12-04T06:41:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294543826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294543827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294543837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294544037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294544037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294544046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/24a571cfc52c4a4db001e8a946d98564 2024-12-04T06:41:24,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4a4089ac34854cb98518b988724980f8 is 50, key is test_row_0/C:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:24,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742046_1222 (size=12301) 2024-12-04T06:41:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294544347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294544347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294544357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:24,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4a4089ac34854cb98518b988724980f8 2024-12-04T06:41:24,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e71625f4e3b541489a4090dfd7a56995 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995 2024-12-04T06:41:24,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995, entries=200, sequenceid=284, filesize=39.0 K 2024-12-04T06:41:24,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/24a571cfc52c4a4db001e8a946d98564 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564 2024-12-04T06:41:24,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T06:41:24,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/4a4089ac34854cb98518b988724980f8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8 2024-12-04T06:41:24,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8, entries=150, sequenceid=284, filesize=12.0 K 2024-12-04T06:41:24,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae9d50f9f78f5f45c22cc72ad2361274 in 1015ms, sequenceid=284, compaction requested=true 2024-12-04T06:41:24,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:24,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:24,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:24,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:24,654 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:24,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:24,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:24,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-04T06:41:24,654 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:24,658 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:24,658 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:24,658 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:24,658 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=100.4 K 2024-12-04T06:41:24,658 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:24,658 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995] 2024-12-04T06:41:24,658 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:24,658 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:24,658 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:24,658 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/6770aab06c844fecaa5cc84097440f6d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.4 K 2024-12-04T06:41:24,659 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da5405d36f742528bd538893750e697, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:24,659 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6770aab06c844fecaa5cc84097440f6d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:24,659 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d4071cc7c44e1d8a41fab304056dcf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733294481281 2024-12-04T06:41:24,660 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9cad6ae8b443128e0e8e6347f9dac0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733294481281 2024-12-04T06:41:24,660 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71625f4e3b541489a4090dfd7a56995, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482480 2024-12-04T06:41:24,660 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a571cfc52c4a4db001e8a946d98564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482489 2024-12-04T06:41:24,677 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:24,678 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/02f1b5a86cd049e4a0ad6b0a0751b392 is 50, key is test_row_0/B:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:24,681 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:24,697 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204dd2432fc5e7640839fcf21be8553e191_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:24,699 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204dd2432fc5e7640839fcf21be8553e191_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:24,699 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204dd2432fc5e7640839fcf21be8553e191_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:24,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742047_1223 (size=12949) 2024-12-04T06:41:24,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/02f1b5a86cd049e4a0ad6b0a0751b392 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/02f1b5a86cd049e4a0ad6b0a0751b392 2024-12-04T06:41:24,731 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into 02f1b5a86cd049e4a0ad6b0a0751b392(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:24,731 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:24,731 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294484653; duration=0sec 2024-12-04T06:41:24,731 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:24,731 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:24,731 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:24,732 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:24,732 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:24,733 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:24,733 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1a524dc312b1484683b06e74fd0e9381, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.4 K 2024-12-04T06:41:24,733 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a524dc312b1484683b06e74fd0e9381, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733294481123 2024-12-04T06:41:24,735 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b7feaccffca34d9cb833bfac9486dd62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1733294481281 2024-12-04T06:41:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742048_1224 (size=4469) 2024-12-04T06:41:24,736 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a4089ac34854cb98518b988724980f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482489 2024-12-04T06:41:24,751 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:24,751 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/143be6bc1687474a9e952a4d0de5729a is 50, key is test_row_0/C:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742049_1225 (size=12949) 2024-12-04T06:41:24,776 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/143be6bc1687474a9e952a4d0de5729a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/143be6bc1687474a9e952a4d0de5729a 2024-12-04T06:41:24,783 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 143be6bc1687474a9e952a4d0de5729a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:24,783 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:24,783 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294484654; duration=0sec 2024-12-04T06:41:24,783 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:24,783 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:24,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T06:41:24,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294483694, max timestamp=1733294484859Num uniques 859; 2024-12-04T06:41:24,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:24,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294483694, max timestamp=1733294484859Num uniques 859; 2024-12-04T06:41:24,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:24,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294483694, max timestamp=1733294484859Num uniques 859; 2024-12-04T06:41:24,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:24,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:24,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:24,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043b0477962a564b129760b200090d8776_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294484859/Put/seqid=0 2024-12-04T06:41:24,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742050_1226 (size=12454) 2024-12-04T06:41:24,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:24,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412043b0477962a564b129760b200090d8776_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043b0477962a564b129760b200090d8776_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:24,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/5c51c7f950b348979aa0f95b96022b9d, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:24,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/5c51c7f950b348979aa0f95b96022b9d is 175, key is test_row_0/A:col10/1733294484859/Put/seqid=0 2024-12-04T06:41:24,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742051_1227 (size=31255) 2024-12-04T06:41:24,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294484948, max timestamp=1733294484966Num uniques 859; 2024-12-04T06:41:24,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:24,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294484948, max timestamp=1733294484966Num uniques 859; 2024-12-04T06:41:24,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294484948, max timestamp=1733294484966Num uniques 859; 2024-12-04T06:41:24,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:24,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:25,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733294484989, max timestamp=1733294485029Num uniques 1068; 2024-12-04T06:41:25,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-04T06:41:25,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733294485012, max timestamp=1733294485029Num uniques 1068; 2024-12-04T06:41:25,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733294485012, max timestamp=1733294485029Num uniques 1068; 2024-12-04T06:41:25,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T06:41:25,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-04T06:41:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733294485073, max timestamp=1733294485082Num uniques 1177; 2024-12-04T06:41:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733294485073, max timestamp=1733294485082Num uniques 1027; 2024-12-04T06:41:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-04T06:41:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T06:41:25,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733294485073, max timestamp=1733294485082Num uniques 1027; 2024-12-04T06:41:25,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-04T06:41:25,137 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#189 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:25,138 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f0198ad2884a4ffc989af8cf5d89c51b is 175, key is test_row_0/A:col10/1733294482490/Put/seqid=0 2024-12-04T06:41:25,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742052_1228 (size=31903) 2024-12-04T06:41:25,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733294485121, max timestamp=1733294485137Num uniques 1086; 2024-12-04T06:41:25,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=8506, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-04T06:41:25,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8498, totalHeapSize=19584, min timestamp=1733294485121, max timestamp=1733294485150Num uniques 1086; 2024-12-04T06:41:25,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49718, new segment data size=8498, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-04T06:41:25,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733294485121, max timestamp=1733294485150Num uniques 1086; 2024-12-04T06:41:25,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49721, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-12-04T06:41:25,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733294485189, max timestamp=1733294485234Num uniques 1045; 2024-12-04T06:41:25,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T06:41:25,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733294485189, max timestamp=1733294485234Num uniques 1045; 2024-12-04T06:41:25,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-04T06:41:25,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733294485189, max timestamp=1733294485234Num uniques 1009; 2024-12-04T06:41:25,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46229, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-12-04T06:41:25,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733294485270, max timestamp=1733294485299Num uniques 1104; 2024-12-04T06:41:25,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-04T06:41:25,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733294485270, max timestamp=1733294485299Num uniques 1104; 2024-12-04T06:41:25,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-12-04T06:41:25,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733294485270, max timestamp=1733294485299Num uniques 1104; 2024-12-04T06:41:25,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-04T06:41:25,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/5c51c7f950b348979aa0f95b96022b9d 2024-12-04T06:41:25,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/299e7959a7a543c2917f6f81c1676279 is 50, key is test_row_0/B:col10/1733294484859/Put/seqid=0 2024-12-04T06:41:25,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742053_1229 (size=12301) 2024-12-04T06:41:25,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/299e7959a7a543c2917f6f81c1676279 2024-12-04T06:41:25,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/e7aeaa733d9f42f2b921e15a83a537e4 is 50, key is test_row_0/C:col10/1733294484859/Put/seqid=0 2024-12-04T06:41:25,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1733294485317, max timestamp=1733294485364Num uniques 1163; 2024-12-04T06:41:25,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=14328, suffix heap size=148920,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=313 2024-12-04T06:41:25,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733294485317, max timestamp=1733294485366Num uniques 1163; 2024-12-04T06:41:25,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=7458, suffix heap size=148920,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=163 2024-12-04T06:41:25,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733294485317, max timestamp=1733294485366Num uniques 1013; 2024-12-04T06:41:25,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-04T06:41:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742054_1230 (size=12301) 2024-12-04T06:41:25,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/e7aeaa733d9f42f2b921e15a83a537e4 2024-12-04T06:41:25,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/5c51c7f950b348979aa0f95b96022b9d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d 2024-12-04T06:41:25,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d, entries=150, sequenceid=308, filesize=30.5 K 2024-12-04T06:41:25,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/299e7959a7a543c2917f6f81c1676279 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279 2024-12-04T06:41:25,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279, entries=150, sequenceid=308, filesize=12.0 K 2024-12-04T06:41:25,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/e7aeaa733d9f42f2b921e15a83a537e4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4 2024-12-04T06:41:25,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733294485397, max timestamp=1733294485423Num uniques 1172; 2024-12-04T06:41:25,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=17032, suffix heap size=149856,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=372 2024-12-04T06:41:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733294485397, max timestamp=1733294485423Num uniques 1022; 2024-12-04T06:41:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=17032, suffix heap size=134256,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=372 2024-12-04T06:41:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1733294485397, max timestamp=1733294485423Num uniques 1022; 2024-12-04T06:41:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=17032, suffix heap size=134256,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=372 2024-12-04T06:41:25,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4, entries=150, sequenceid=308, filesize=12.0 K 2024-12-04T06:41:25,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=73.80 KB/75570 for ae9d50f9f78f5f45c22cc72ad2361274 in 576ms, sequenceid=308, compaction requested=false 2024-12-04T06:41:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:25,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=80.51 KB heapSize=205.64 KB 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:25,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c19765550adc44e890dc7ec06176df5d_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:25,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742055_1231 (size=14994) 2024-12-04T06:41:25,478 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:25,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204c19765550adc44e890dc7ec06176df5d_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c19765550adc44e890dc7ec06176df5d_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:25,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b4e563a3e4ee44c29e75e765e04bd258, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:25,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b4e563a3e4ee44c29e75e765e04bd258 is 175, key is test_row_0/A:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-04T06:41:25,488 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:25,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-04T06:41:25,490 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:25,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:25,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742056_1232 (size=39949) 2024-12-04T06:41:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294545505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294545506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294545511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,560 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/f0198ad2884a4ffc989af8cf5d89c51b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b 2024-12-04T06:41:25,569 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into f0198ad2884a4ffc989af8cf5d89c51b(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:25,569 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:25,569 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294484653; duration=0sec 2024-12-04T06:41:25,569 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:25,569 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:25,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294545627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294545627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294545627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:25,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294545836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294545836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294545847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:25,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:25,901 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b4e563a3e4ee44c29e75e765e04bd258 2024-12-04T06:41:25,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a25277cb33104f698f63dbb5012a27cf is 50, key is test_row_0/B:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:25,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742057_1233 (size=12301) 2024-12-04T06:41:25,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:25,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:26,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294546146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294546146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294546167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a25277cb33104f698f63dbb5012a27cf 2024-12-04T06:41:26,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1c70610e844949ed91e410d468fd00c8 is 50, key is test_row_0/C:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:26,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742058_1234 (size=12301) 2024-12-04T06:41:26,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:26,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294546657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294546657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294546686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:26,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:26,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:26,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1c70610e844949ed91e410d468fd00c8 2024-12-04T06:41:26,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/b4e563a3e4ee44c29e75e765e04bd258 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258 2024-12-04T06:41:26,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258, entries=200, sequenceid=453, filesize=39.0 K 2024-12-04T06:41:26,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/a25277cb33104f698f63dbb5012a27cf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf 2024-12-04T06:41:26,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf, entries=150, sequenceid=453, filesize=12.0 K 2024-12-04T06:41:26,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1c70610e844949ed91e410d468fd00c8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8 2024-12-04T06:41:26,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8, entries=150, sequenceid=453, filesize=12.0 K 2024-12-04T06:41:26,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~205.59 KB/210528, currentSize=107.34 KB/109920 for ae9d50f9f78f5f45c22cc72ad2361274 in 1324ms, sequenceid=453, compaction requested=true 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:26,766 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:26,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:26,767 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:26,767 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:26,767 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:26,767 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:26,767 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:26,767 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,768 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,768 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=100.7 K 2024-12-04T06:41:26,768 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/02f1b5a86cd049e4a0ad6b0a0751b392, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.7 K 2024-12-04T06:41:26,768 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,768 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258] 2024-12-04T06:41:26,768 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0198ad2884a4ffc989af8cf5d89c51b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482489 2024-12-04T06:41:26,768 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f1b5a86cd049e4a0ad6b0a0751b392, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482489 2024-12-04T06:41:26,769 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 299e7959a7a543c2917f6f81c1676279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733294483709 2024-12-04T06:41:26,769 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c51c7f950b348979aa0f95b96022b9d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733294483709 2024-12-04T06:41:26,769 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a25277cb33104f698f63dbb5012a27cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:26,769 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4e563a3e4ee44c29e75e765e04bd258, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:26,811 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:26,813 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:26,813 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef is 50, key is test_row_0/B:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:26,821 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204e07d1ae79c6b408ebdf28a01fb494c2b_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:26,824 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204e07d1ae79c6b408ebdf28a01fb494c2b_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:26,824 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e07d1ae79c6b408ebdf28a01fb494c2b_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742059_1235 (size=13051) 2024-12-04T06:41:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742060_1236 (size=4469) 2024-12-04T06:41:26,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-04T06:41:26,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:26,889 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204407edd5e25d54bf98e94a0b84244e7b9_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294485504/Put/seqid=0 2024-12-04T06:41:26,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742061_1237 (size=12454) 2024-12-04T06:41:27,244 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#198 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:27,245 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/031336f2472a4cff853a3d46dee7b41f is 175, key is test_row_0/A:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:27,247 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef 2024-12-04T06:41:27,255 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into ab449cb5b1c4431f84b7d0a4e3f1c4ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:27,255 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:27,255 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294486766; duration=0sec 2024-12-04T06:41:27,255 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:27,255 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:27,255 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:27,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:27,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:27,257 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:27,257 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/143be6bc1687474a9e952a4d0de5729a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.7 K 2024-12-04T06:41:27,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742062_1238 (size=32005) 2024-12-04T06:41:27,258 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 143be6bc1687474a9e952a4d0de5729a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733294482489 2024-12-04T06:41:27,259 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e7aeaa733d9f42f2b921e15a83a537e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733294483709 2024-12-04T06:41:27,259 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c70610e844949ed91e410d468fd00c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:27,280 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#200 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:27,281 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/485834cec6a4476697b0fb75ba0afb6c is 50, key is test_row_0/C:col10/1733294485441/Put/seqid=0 2024-12-04T06:41:27,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742063_1239 (size=13051) 2024-12-04T06:41:27,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204407edd5e25d54bf98e94a0b84244e7b9_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204407edd5e25d54bf98e94a0b84244e7b9_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/58e4d8a398f94df5a3d76de21ec90904, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/58e4d8a398f94df5a3d76de21ec90904 is 175, key is test_row_0/A:col10/1733294485504/Put/seqid=0 2024-12-04T06:41:27,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742064_1240 (size=31255) 2024-12-04T06:41:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:27,665 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/031336f2472a4cff853a3d46dee7b41f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f 2024-12-04T06:41:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:27,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. as already flushing 2024-12-04T06:41:27,673 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into 031336f2472a4cff853a3d46dee7b41f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:27,673 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:27,673 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294486766; duration=0sec 2024-12-04T06:41:27,673 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:27,673 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:27,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294547694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:27,699 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/485834cec6a4476697b0fb75ba0afb6c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/485834cec6a4476697b0fb75ba0afb6c 2024-12-04T06:41:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294547696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:27,707 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 485834cec6a4476697b0fb75ba0afb6c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:27,707 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:27,707 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294486766; duration=0sec 2024-12-04T06:41:27,707 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:27,707 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:27,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294547707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:27,710 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:27,748 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/58e4d8a398f94df5a3d76de21ec90904 2024-12-04T06:41:27,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b1fcf296be7f42068fe1bd74ef4b1a8f is 50, key is test_row_0/B:col10/1733294485504/Put/seqid=0 2024-12-04T06:41:27,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742065_1241 (size=12301) 2024-12-04T06:41:27,783 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b1fcf296be7f42068fe1bd74ef4b1a8f 2024-12-04T06:41:27,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d95effad75d34413a02541085993e8af is 50, key is test_row_0/C:col10/1733294485504/Put/seqid=0 2024-12-04T06:41:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294547807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294547807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:27,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742066_1242 (size=12301) 2024-12-04T06:41:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294548016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294548016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,229 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d95effad75d34413a02541085993e8af 2024-12-04T06:41:28,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/58e4d8a398f94df5a3d76de21ec90904 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904 2024-12-04T06:41:28,247 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904, entries=150, sequenceid=473, filesize=30.5 K 2024-12-04T06:41:28,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/b1fcf296be7f42068fe1bd74ef4b1a8f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f 2024-12-04T06:41:28,256 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f, entries=150, sequenceid=473, filesize=12.0 K 2024-12-04T06:41:28,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/d95effad75d34413a02541085993e8af as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af 2024-12-04T06:41:28,267 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af, entries=150, sequenceid=473, filesize=12.0 K 2024-12-04T06:41:28,269 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ae9d50f9f78f5f45c22cc72ad2361274 in 1380ms, sequenceid=473, compaction requested=false 2024-12-04T06:41:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T06:41:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T06:41:28,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T06:41:28,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7730 sec 2024-12-04T06:41:28,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7850 sec 2024-12-04T06:41:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:28,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T06:41:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:28,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b1515813577e4e51a52da83983b5b64f_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742067_1243 (size=14994) 2024-12-04T06:41:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294548371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294548375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294548487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294548487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294548697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294548697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:28,716 INFO [Thread-730 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-730. 2024-12-04T06:41:28,716 INFO [Thread-728 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-728. 2024-12-04T06:41:28,716 DEBUG [Thread-730 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:28,716 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,716 DEBUG [Thread-728 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:28,716 DEBUG [Thread-728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,716 DEBUG [Thread-730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,716 DEBUG [Thread-728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:28,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:28,717 INFO [Thread-726 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-726. 2024-12-04T06:41:28,717 DEBUG [Thread-726 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:28,717 DEBUG [Thread-726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,717 DEBUG [Thread-726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:28,724 INFO [Thread-732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-732. 2024-12-04T06:41:28,724 DEBUG [Thread-732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:28,724 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,724 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:28,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:28,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:28,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b1515813577e4e51a52da83983b5b64f_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b1515813577e4e51a52da83983b5b64f_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:28,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e5b9f4dcb637446e910c9627ad07c22c, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:28,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e5b9f4dcb637446e910c9627ad07c22c is 175, key is test_row_0/A:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:28,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742068_1244 (size=39949) 2024-12-04T06:41:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294549007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294549007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,081 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:41:29,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e5b9f4dcb637446e910c9627ad07c22c 2024-12-04T06:41:29,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/42a0ffec4905469c9086890a1fc038d0 is 50, key is test_row_0/B:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:29,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742069_1245 (size=12301) 2024-12-04T06:41:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43610 deadline: 1733294549517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43620 deadline: 1733294549517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/42a0ffec4905469c9086890a1fc038d0 2024-12-04T06:41:29,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d77e70e825c4a2980d8202c4c086d69 is 50, key is test_row_0/C:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:29,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742070_1246 (size=12301) 2024-12-04T06:41:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43632 deadline: 1733294549616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,617 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T06:41:29,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43636 deadline: 1733294549667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,668 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:29,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43650 deadline: 1733294549737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:29,738 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in ae9d50f9f78f5f45c22cc72ad2361274 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:41:30,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d77e70e825c4a2980d8202c4c086d69 2024-12-04T06:41:30,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/e5b9f4dcb637446e910c9627ad07c22c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c 2024-12-04T06:41:30,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c, entries=200, sequenceid=491, filesize=39.0 K 2024-12-04T06:41:30,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/42a0ffec4905469c9086890a1fc038d0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0 2024-12-04T06:41:30,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0, entries=150, sequenceid=491, filesize=12.0 K 2024-12-04T06:41:30,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/7d77e70e825c4a2980d8202c4c086d69 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69 2024-12-04T06:41:30,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69, entries=150, sequenceid=491, filesize=12.0 K 2024-12-04T06:41:30,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ae9d50f9f78f5f45c22cc72ad2361274 in 1701ms, sequenceid=491, compaction requested=true 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:30,031 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae9d50f9f78f5f45c22cc72ad2361274:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:30,031 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:30,032 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:30,032 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/A is initiating minor compaction (all files) 2024-12-04T06:41:30,032 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:30,032 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/A in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:30,032 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=100.8 K 2024-12-04T06:41:30,032 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:30,032 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c] 2024-12-04T06:41:30,032 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/B is initiating minor compaction (all files) 2024-12-04T06:41:30,032 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/B in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:30,032 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.8 K 2024-12-04T06:41:30,033 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 031336f2472a4cff853a3d46dee7b41f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:30,033 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab449cb5b1c4431f84b7d0a4e3f1c4ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:30,033 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e4d8a398f94df5a3d76de21ec90904, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733294485487 2024-12-04T06:41:30,033 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fcf296be7f42068fe1bd74ef4b1a8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733294485487 2024-12-04T06:41:30,033 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b9f4dcb637446e910c9627ad07c22c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733294487692 2024-12-04T06:41:30,034 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42a0ffec4905469c9086890a1fc038d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733294487694 2024-12-04T06:41:30,042 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:30,042 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#B#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:30,043 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/c12770a66203406e8758330cc2cd97b3 is 50, key is test_row_0/B:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:30,043 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120463b95cb89e18415aa5482b85a3438c8e_ae9d50f9f78f5f45c22cc72ad2361274 store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:30,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742071_1247 (size=13153) 2024-12-04T06:41:30,053 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120463b95cb89e18415aa5482b85a3438c8e_ae9d50f9f78f5f45c22cc72ad2361274, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:30,054 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120463b95cb89e18415aa5482b85a3438c8e_ae9d50f9f78f5f45c22cc72ad2361274 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:30,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742072_1248 (size=4469) 2024-12-04T06:41:30,058 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#A#compaction#207 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:30,058 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/13868153d21841bca4535b97351cddfc is 175, key is test_row_0/A:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:30,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742073_1249 (size=32107) 2024-12-04T06:41:30,451 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/c12770a66203406e8758330cc2cd97b3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c12770a66203406e8758330cc2cd97b3 2024-12-04T06:41:30,455 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/B of ae9d50f9f78f5f45c22cc72ad2361274 into c12770a66203406e8758330cc2cd97b3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:30,455 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:30,455 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/B, priority=13, startTime=1733294490031; duration=0sec 2024-12-04T06:41:30,455 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:30,455 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:B 2024-12-04T06:41:30,455 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:30,456 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:30,456 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ae9d50f9f78f5f45c22cc72ad2361274/C is initiating minor compaction (all files) 2024-12-04T06:41:30,456 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae9d50f9f78f5f45c22cc72ad2361274/C in TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:30,456 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/485834cec6a4476697b0fb75ba0afb6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp, totalSize=36.8 K 2024-12-04T06:41:30,457 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485834cec6a4476697b0fb75ba0afb6c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733294485423 2024-12-04T06:41:30,457 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95effad75d34413a02541085993e8af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733294485487 2024-12-04T06:41:30,457 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d77e70e825c4a2980d8202c4c086d69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733294487694 2024-12-04T06:41:30,464 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae9d50f9f78f5f45c22cc72ad2361274#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:30,464 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/67701c447a56447782b6c494680b4747 is 50, key is test_row_0/C:col10/1733294488328/Put/seqid=0 2024-12-04T06:41:30,466 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/13868153d21841bca4535b97351cddfc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/13868153d21841bca4535b97351cddfc 2024-12-04T06:41:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742074_1250 (size=13153) 2024-12-04T06:41:30,470 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/A of ae9d50f9f78f5f45c22cc72ad2361274 into 13868153d21841bca4535b97351cddfc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:30,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:30,470 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/A, priority=13, startTime=1733294490031; duration=0sec 2024-12-04T06:41:30,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:30,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:A 2024-12-04T06:41:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:30,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:41:30,529 INFO [Thread-717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-717. 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:30,529 DEBUG [Thread-717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:30,529 INFO [Thread-719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-719. 2024-12-04T06:41:30,529 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:30,529 DEBUG [Thread-719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:30,529 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:30,529 DEBUG [Thread-719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:30,529 DEBUG [Thread-717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:30,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:30,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:30,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:30,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045b318c1fef5d484eac0003442beb5874_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294490527/Put/seqid=0 2024-12-04T06:41:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742075_1251 (size=12454) 2024-12-04T06:41:30,873 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/67701c447a56447782b6c494680b4747 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/67701c447a56447782b6c494680b4747 2024-12-04T06:41:30,878 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae9d50f9f78f5f45c22cc72ad2361274/C of ae9d50f9f78f5f45c22cc72ad2361274 into 67701c447a56447782b6c494680b4747(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:30,878 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:30,878 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., storeName=ae9d50f9f78f5f45c22cc72ad2361274/C, priority=13, startTime=1733294490031; duration=0sec 2024-12-04T06:41:30,878 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:30,878 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae9d50f9f78f5f45c22cc72ad2361274:C 2024-12-04T06:41:30,939 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:30,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045b318c1fef5d484eac0003442beb5874_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045b318c1fef5d484eac0003442beb5874_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:30,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/96a5da1e90294efa80409ae62b0b533b, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:30,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/96a5da1e90294efa80409ae62b0b533b is 175, key is test_row_0/A:col10/1733294490527/Put/seqid=0 2024-12-04T06:41:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742076_1252 (size=31255) 2024-12-04T06:41:31,348 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/96a5da1e90294efa80409ae62b0b533b 2024-12-04T06:41:31,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/67d62c3889274d1daf04438d1a0c9449 is 50, key is test_row_0/B:col10/1733294490527/Put/seqid=0 2024-12-04T06:41:31,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742077_1253 (size=12301) 2024-12-04T06:41:31,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/67d62c3889274d1daf04438d1a0c9449 2024-12-04T06:41:31,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/75b24a4cc526473991e750f65b0108a2 is 50, key is test_row_0/C:col10/1733294490527/Put/seqid=0 2024-12-04T06:41:31,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742078_1254 (size=12301) 2024-12-04T06:41:32,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/75b24a4cc526473991e750f65b0108a2 2024-12-04T06:41:32,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/96a5da1e90294efa80409ae62b0b533b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/96a5da1e90294efa80409ae62b0b533b 2024-12-04T06:41:32,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/96a5da1e90294efa80409ae62b0b533b, entries=150, sequenceid=512, filesize=30.5 K 2024-12-04T06:41:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/67d62c3889274d1daf04438d1a0c9449 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/67d62c3889274d1daf04438d1a0c9449 2024-12-04T06:41:32,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/67d62c3889274d1daf04438d1a0c9449, entries=150, sequenceid=512, filesize=12.0 K 2024-12-04T06:41:32,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/75b24a4cc526473991e750f65b0108a2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/75b24a4cc526473991e750f65b0108a2 2024-12-04T06:41:32,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/75b24a4cc526473991e750f65b0108a2, entries=150, sequenceid=512, filesize=12.0 K 2024-12-04T06:41:32,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for ae9d50f9f78f5f45c22cc72ad2361274 in 1659ms, sequenceid=512, compaction requested=false 2024-12-04T06:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae9d50f9f78f5f45c22cc72ad2361274: 2024-12-04T06:41:33,778 INFO [Thread-723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-723. 2024-12-04T06:41:33,778 DEBUG [Thread-723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:33,778 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:33,778 DEBUG [Thread-723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:33,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:39,648 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-12-04T06:41:39,648 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:39,648 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,648 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:39,688 INFO [Thread-721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-721. 2024-12-04T06:41:39,688 DEBUG [Thread-721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:41:39,688 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,688 DEBUG [Thread-721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:41:39,688 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2516 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2578 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1111 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3333 rows 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-12-04T06:41:39,689 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3336 rows 2024-12-04T06:41:39,689 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:41:39,689 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:41:39,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:39,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:41:39,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:41:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:39,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294499692"}]},"ts":"1733294499692"} 2024-12-04T06:41:39,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:41:39,694 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:41:39,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:41:39,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, UNASSIGN}] 2024-12-04T06:41:39,696 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, UNASSIGN 2024-12-04T06:41:39,697 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:39,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, UNASSIGN because future has completed 2024-12-04T06:41:39,698 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:41:39,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:39,850 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:39,850 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing ae9d50f9f78f5f45c22cc72ad2361274, disabling compactions & flushes 2024-12-04T06:41:39,851 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. after waiting 0 ms 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:39,851 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing ae9d50f9f78f5f45c22cc72ad2361274 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=A 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=B 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae9d50f9f78f5f45c22cc72ad2361274, store=C 2024-12-04T06:41:39,851 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:39,856 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120443cf3304ce9b40df806c74682018451d_ae9d50f9f78f5f45c22cc72ad2361274 is 50, key is test_row_0/A:col10/1733294499687/Put/seqid=0 2024-12-04T06:41:39,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742079_1255 (size=9914) 2024-12-04T06:41:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:40,261 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:40,265 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120443cf3304ce9b40df806c74682018451d_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443cf3304ce9b40df806c74682018451d_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:40,266 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/62f49058effd4e919af48e1a9d7e647a, store: [table=TestAcidGuarantees family=A region=ae9d50f9f78f5f45c22cc72ad2361274] 2024-12-04T06:41:40,267 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/62f49058effd4e919af48e1a9d7e647a is 175, key is test_row_0/A:col10/1733294499687/Put/seqid=0 2024-12-04T06:41:40,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742080_1256 (size=22561) 2024-12-04T06:41:40,271 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=519, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/62f49058effd4e919af48e1a9d7e647a 2024-12-04T06:41:40,278 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/c836c7332089474a9a270eaa44d5f472 is 50, key is test_row_0/B:col10/1733294499687/Put/seqid=0 2024-12-04T06:41:40,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742081_1257 (size=9857) 2024-12-04T06:41:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:40,688 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/c836c7332089474a9a270eaa44d5f472 2024-12-04T06:41:40,697 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1d0d84dde4874e3d8c1ce717020ec88e is 50, key is test_row_0/C:col10/1733294499687/Put/seqid=0 2024-12-04T06:41:40,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742082_1258 (size=9857) 2024-12-04T06:41:40,703 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1d0d84dde4874e3d8c1ce717020ec88e 2024-12-04T06:41:40,713 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/A/62f49058effd4e919af48e1a9d7e647a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/62f49058effd4e919af48e1a9d7e647a 2024-12-04T06:41:40,718 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/62f49058effd4e919af48e1a9d7e647a, entries=100, sequenceid=519, filesize=22.0 K 2024-12-04T06:41:40,719 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/B/c836c7332089474a9a270eaa44d5f472 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c836c7332089474a9a270eaa44d5f472 2024-12-04T06:41:40,724 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c836c7332089474a9a270eaa44d5f472, entries=100, sequenceid=519, filesize=9.6 K 2024-12-04T06:41:40,725 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/.tmp/C/1d0d84dde4874e3d8c1ce717020ec88e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1d0d84dde4874e3d8c1ce717020ec88e 2024-12-04T06:41:40,731 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1d0d84dde4874e3d8c1ce717020ec88e, entries=100, sequenceid=519, filesize=9.6 K 2024-12-04T06:41:40,732 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ae9d50f9f78f5f45c22cc72ad2361274 in 880ms, sequenceid=519, compaction requested=true 2024-12-04T06:41:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c] to archive 2024-12-04T06:41:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f70f90827dc2417d94928c8868d56429 2024-12-04T06:41:40,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/daf4a2127dd64c39be73fa7a42be3133 2024-12-04T06:41:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b79689c38dc64d069d6f535cb19a5fb2 2024-12-04T06:41:40,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f4b2a5f54e5847fcbb5cb7333661030a 2024-12-04T06:41:40,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1c7bd97a48be4baab5eaf113626db1c2 2024-12-04T06:41:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/dceda970b6d8477fa74e0bb42bc8b16c 2024-12-04T06:41:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/462e6223d7154f8681ca2b1c6e290cdd 2024-12-04T06:41:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/8289ae78d40b427f95b4b8722575a3cc 2024-12-04T06:41:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/d8e1264551464d538299c4ccb7f1419a 2024-12-04T06:41:40,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b04d42fe45094f23b70f2fa582e5dff2 2024-12-04T06:41:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c9b417f32018436097512af86ee8dfdf 2024-12-04T06:41:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8bff16f840940abadad8b820bff7bf2 2024-12-04T06:41:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/a8b58802b54148c3b4a91df4c6670ec0 2024-12-04T06:41:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/bd0dc3f151ec40cba8c2b981315d3f4d 2024-12-04T06:41:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/c78a3725dbab45cb90cf9cc781186efb 2024-12-04T06:41:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1ea7dcf0256c461697e8f4a509ef399e 2024-12-04T06:41:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/1da5405d36f742528bd538893750e697 2024-12-04T06:41:40,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f6d4071cc7c44e1d8a41fab304056dcf 2024-12-04T06:41:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e71625f4e3b541489a4090dfd7a56995 2024-12-04T06:41:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/f0198ad2884a4ffc989af8cf5d89c51b 2024-12-04T06:41:40,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/5c51c7f950b348979aa0f95b96022b9d 2024-12-04T06:41:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/b4e563a3e4ee44c29e75e765e04bd258 2024-12-04T06:41:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/031336f2472a4cff853a3d46dee7b41f 2024-12-04T06:41:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/58e4d8a398f94df5a3d76de21ec90904 2024-12-04T06:41:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/e5b9f4dcb637446e910c9627ad07c22c 2024-12-04T06:41:40,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/8d1d1eba1d5d476fb79a0023743a39f6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f40849c3d9cb46c2997a8e0210f12232, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/fe078e2b62ef4cf29044a86bf088fb74, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/46a9cb50d0cb4785b193eb53d26d4d4c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/6770aab06c844fecaa5cc84097440f6d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/02f1b5a86cd049e4a0ad6b0a0751b392, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0] to archive 2024-12-04T06:41:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/7d284a1280b0435da88a629d71857a09 2024-12-04T06:41:40,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4454b7c9b23445b69d8cfa3b8f0684cf 2024-12-04T06:41:40,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/8d1d1eba1d5d476fb79a0023743a39f6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/8d1d1eba1d5d476fb79a0023743a39f6 2024-12-04T06:41:40,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1b0ee0494dbc4d2db1c276db6825397d 2024-12-04T06:41:40,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/4710eda8d912420e992d56784fee74bb 2024-12-04T06:41:40,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f40849c3d9cb46c2997a8e0210f12232 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f40849c3d9cb46c2997a8e0210f12232 2024-12-04T06:41:40,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/93d82afa74a649978673cb56d9d7db91 2024-12-04T06:41:40,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/bcdd57c8f8624f3a943c8a6f35b6d726 2024-12-04T06:41:40,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/fe078e2b62ef4cf29044a86bf088fb74 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/fe078e2b62ef4cf29044a86bf088fb74 2024-12-04T06:41:40,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/28e105d74eeb427fa1c24de588368b13 2024-12-04T06:41:40,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/0b5581c8f6e9430e8cc4bde0831c8e85 2024-12-04T06:41:40,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a2ef9b2cd51d4de1a3eaceeb5c68fcaa 2024-12-04T06:41:40,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/46a9cb50d0cb4785b193eb53d26d4d4c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/46a9cb50d0cb4785b193eb53d26d4d4c 2024-12-04T06:41:40,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b4622628a82448f8aaf23f82a7941201 2024-12-04T06:41:40,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/d6df83896e794b8fbe1ba47ff232cf52 2024-12-04T06:41:40,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/6770aab06c844fecaa5cc84097440f6d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/6770aab06c844fecaa5cc84097440f6d 2024-12-04T06:41:40,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/f8584a7ec47d4267b491a7266309b2b4 2024-12-04T06:41:40,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/1a9cad6ae8b443128e0e8e6347f9dac0 2024-12-04T06:41:40,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/02f1b5a86cd049e4a0ad6b0a0751b392 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/02f1b5a86cd049e4a0ad6b0a0751b392 2024-12-04T06:41:40,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/24a571cfc52c4a4db001e8a946d98564 2024-12-04T06:41:40,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/299e7959a7a543c2917f6f81c1676279 2024-12-04T06:41:40,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/ab449cb5b1c4431f84b7d0a4e3f1c4ef 2024-12-04T06:41:40,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/a25277cb33104f698f63dbb5012a27cf 2024-12-04T06:41:40,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/b1fcf296be7f42068fe1bd74ef4b1a8f 2024-12-04T06:41:40,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/42a0ffec4905469c9086890a1fc038d0 2024-12-04T06:41:40,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d2f6fa13db19452ab2766354f6316fb2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d688f85d13d247fbbed1b3ac6f861e97, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/2aa45fb43c564e09b134b99e9f4f4bb9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7c15c202ace442798ca4f5fdb575ea07, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1a524dc312b1484683b06e74fd0e9381, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/143be6bc1687474a9e952a4d0de5729a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/485834cec6a4476697b0fb75ba0afb6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69] to archive 2024-12-04T06:41:40,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:41:40,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4934cb46a7da4cf18f5fec12670eabef 2024-12-04T06:41:40,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d53eea84d6f4c109f528f10e7606476 2024-12-04T06:41:40,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d2f6fa13db19452ab2766354f6316fb2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d2f6fa13db19452ab2766354f6316fb2 2024-12-04T06:41:40,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4d62004b8796405f9814defb42559eb5 2024-12-04T06:41:40,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/f4de3a98d3c345c69b99384483821bf1 2024-12-04T06:41:40,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d688f85d13d247fbbed1b3ac6f861e97 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d688f85d13d247fbbed1b3ac6f861e97 2024-12-04T06:41:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:40,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/68e0a57e1c75400ab2f21e351f383bcd 2024-12-04T06:41:40,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/962f1a62c3df4e5f9bf77e2240fe986d 2024-12-04T06:41:40,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/2aa45fb43c564e09b134b99e9f4f4bb9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/2aa45fb43c564e09b134b99e9f4f4bb9 2024-12-04T06:41:40,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/951d2ba6c6454ef1a0ddcf6a99de3c4a 2024-12-04T06:41:40,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8bed6e7fde224b18bfa7eeb93382417e 2024-12-04T06:41:40,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4904892f6038487783b0b57e2f8cb538 2024-12-04T06:41:40,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7c15c202ace442798ca4f5fdb575ea07 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7c15c202ace442798ca4f5fdb575ea07 2024-12-04T06:41:40,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/55c01f03b7ed47739bfe7171c9113503 2024-12-04T06:41:40,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/8a971fa0f28a44d3a8a61e2c629a4ffd 2024-12-04T06:41:40,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1a524dc312b1484683b06e74fd0e9381 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1a524dc312b1484683b06e74fd0e9381 2024-12-04T06:41:40,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d5b184cecdcc471393f680c72bee8e44 2024-12-04T06:41:40,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/b7feaccffca34d9cb833bfac9486dd62 2024-12-04T06:41:40,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/143be6bc1687474a9e952a4d0de5729a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/143be6bc1687474a9e952a4d0de5729a 2024-12-04T06:41:40,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/4a4089ac34854cb98518b988724980f8 2024-12-04T06:41:40,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/e7aeaa733d9f42f2b921e15a83a537e4 2024-12-04T06:41:40,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/485834cec6a4476697b0fb75ba0afb6c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/485834cec6a4476697b0fb75ba0afb6c 2024-12-04T06:41:40,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1c70610e844949ed91e410d468fd00c8 2024-12-04T06:41:40,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/d95effad75d34413a02541085993e8af 2024-12-04T06:41:40,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/7d77e70e825c4a2980d8202c4c086d69 2024-12-04T06:41:40,863 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits/522.seqid, newMaxSeqId=522, maxSeqId=4 2024-12-04T06:41:40,864 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. 2024-12-04T06:41:40,864 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for ae9d50f9f78f5f45c22cc72ad2361274: Waiting for close lock at 1733294499851Running coprocessor pre-close hooks at 1733294499851Disabling compacts and flushes for region at 1733294499851Disabling writes for close at 1733294499851Obtaining lock to block concurrent updates at 1733294499851Preparing flush snapshotting stores in ae9d50f9f78f5f45c22cc72ad2361274 at 1733294499851Finished memstore snapshotting TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733294499851Flushing stores of TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274. at 1733294499852 (+1 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/A: creating writer at 1733294499852Flushing mob file ae9d50f9f78f5f45c22cc72ad2361274/A: appending metadata at 1733294499856 (+4 ms)Flushing mob file ae9d50f9f78f5f45c22cc72ad2361274/A: closing flushed file at 1733294499856Flushing ae9d50f9f78f5f45c22cc72ad2361274/A: appending metadata at 1733294500266 (+410 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/A: closing flushed file at 1733294500266Flushing ae9d50f9f78f5f45c22cc72ad2361274/B: creating writer at 1733294500275 (+9 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/B: appending metadata at 1733294500278 (+3 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/B: closing flushed file at 1733294500278Flushing ae9d50f9f78f5f45c22cc72ad2361274/C: creating writer at 1733294500693 (+415 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/C: appending metadata at 1733294500696 (+3 ms)Flushing ae9d50f9f78f5f45c22cc72ad2361274/C: closing flushed file at 1733294500696Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195bd440: reopening flushed file at 1733294500708 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8cd3f5e: reopening flushed file at 1733294500718 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eb3e699: reopening flushed file at 1733294500724 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ae9d50f9f78f5f45c22cc72ad2361274 in 880ms, sequenceid=519, compaction requested=true at 1733294500732 (+8 ms)Writing region close event to WAL at 1733294500860 (+128 ms)Running coprocessor post-close hooks at 1733294500864 (+4 ms)Closed at 1733294500864 2024-12-04T06:41:40,867 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:40,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=ae9d50f9f78f5f45c22cc72ad2361274, regionState=CLOSED 2024-12-04T06:41:40,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:40,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T06:41:40,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure ae9d50f9f78f5f45c22cc72ad2361274, server=607fd5c6574c,38257,1733294430862 in 1.1720 sec 2024-12-04T06:41:40,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-04T06:41:40,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae9d50f9f78f5f45c22cc72ad2361274, UNASSIGN in 1.1760 sec 2024-12-04T06:41:40,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T06:41:40,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1800 sec 2024-12-04T06:41:40,878 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294500877"}]},"ts":"1733294500877"} 2024-12-04T06:41:40,879 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:41:40,879 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:41:40,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-12-04T06:41:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T06:41:41,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:41,837 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:41:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T06:41:41,840 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,843 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,844 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits] 2024-12-04T06:41:41,846 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/13868153d21841bca4535b97351cddfc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/13868153d21841bca4535b97351cddfc 2024-12-04T06:41:41,847 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/62f49058effd4e919af48e1a9d7e647a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/62f49058effd4e919af48e1a9d7e647a 2024-12-04T06:41:41,848 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/96a5da1e90294efa80409ae62b0b533b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/A/96a5da1e90294efa80409ae62b0b533b 2024-12-04T06:41:41,850 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/67d62c3889274d1daf04438d1a0c9449 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/67d62c3889274d1daf04438d1a0c9449 2024-12-04T06:41:41,851 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c12770a66203406e8758330cc2cd97b3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c12770a66203406e8758330cc2cd97b3 2024-12-04T06:41:41,852 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c836c7332089474a9a270eaa44d5f472 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/B/c836c7332089474a9a270eaa44d5f472 2024-12-04T06:41:41,854 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1d0d84dde4874e3d8c1ce717020ec88e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/1d0d84dde4874e3d8c1ce717020ec88e 2024-12-04T06:41:41,855 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/67701c447a56447782b6c494680b4747 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/67701c447a56447782b6c494680b4747 2024-12-04T06:41:41,856 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/75b24a4cc526473991e750f65b0108a2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/C/75b24a4cc526473991e750f65b0108a2 2024-12-04T06:41:41,858 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits/522.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274/recovered.edits/522.seqid 2024-12-04T06:41:41,859 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,859 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:41:41,860 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:41:41,860 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T06:41:41,863 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204014061710150461cbe96e4413d9b76e8_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204014061710150461cbe96e4413d9b76e8_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,864 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040577706224b34d7db7ce4b936d12fce7_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040577706224b34d7db7ce4b936d12fce7_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,866 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fde76ca4a5a4a61a4e1ce9531c6c289_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412040fde76ca4a5a4a61a4e1ce9531c6c289_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,867 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042cfbc9f7390e4823bdaf638c2ac38fa3_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412042cfbc9f7390e4823bdaf638c2ac38fa3_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,868 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043b0477962a564b129760b200090d8776_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043b0477962a564b129760b200090d8776_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,869 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043ffc5a9897dc43d1b62fe9abd2bc39d1_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412043ffc5a9897dc43d1b62fe9abd2bc39d1_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,870 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204407edd5e25d54bf98e94a0b84244e7b9_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204407edd5e25d54bf98e94a0b84244e7b9_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,871 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443cf3304ce9b40df806c74682018451d_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120443cf3304ce9b40df806c74682018451d_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204451342bffab54ea1a5b24b3baf74683b_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204451342bffab54ea1a5b24b3baf74683b_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c67bf39c8084da48d881628eeed2467_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412044c67bf39c8084da48d881628eeed2467_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204552c029e141e41b2a13f830b657a51c2_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204552c029e141e41b2a13f830b657a51c2_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045b318c1fef5d484eac0003442beb5874_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045b318c1fef5d484eac0003442beb5874_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,876 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120468df178b6803491eaaa3eae1957b62be_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120468df178b6803491eaaa3eae1957b62be_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487d57eb661ff4ab48b1d7acd3e3c8e4b_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120487d57eb661ff4ab48b1d7acd3e3c8e4b_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049cc0bf70dbee475ebeb5a138eae9bd36_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049cc0bf70dbee475ebeb5a138eae9bd36_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b1515813577e4e51a52da83983b5b64f_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b1515813577e4e51a52da83983b5b64f_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c19765550adc44e890dc7ec06176df5d_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c19765550adc44e890dc7ec06176df5d_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c46fdff4178a4c0c8681d6042c76c8a4_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204c46fdff4178a4c0c8681d6042c76c8a4_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204db570c6273fe44a6936a67eb1a0cc540_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204db570c6273fe44a6936a67eb1a0cc540_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ede2e8ee900242e3a20f8de656d48c3f_ae9d50f9f78f5f45c22cc72ad2361274 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ede2e8ee900242e3a20f8de656d48c3f_ae9d50f9f78f5f45c22cc72ad2361274 2024-12-04T06:41:41,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:41:41,887 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,889 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:41:41,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:41:41,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:41:41,893 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294501892"}]},"ts":"9223372036854775807"} 2024-12-04T06:41:41,894 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:41:41,894 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae9d50f9f78f5f45c22cc72ad2361274, NAME => 'TestAcidGuarantees,,1733294466546.ae9d50f9f78f5f45c22cc72ad2361274.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:41:41,895 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:41:41,895 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294501895"}]},"ts":"9223372036854775807"} 2024-12-04T06:41:41,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:41:41,897 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-12-04T06:41:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T06:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:41:41,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:41,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-316354366_22 at /127.0.0.1:49452 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 379) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6545 (was 7144) 2024-12-04T06:41:41,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=6544 2024-12-04T06:41:41,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:41:41,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:41:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:41:41,974 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:41:41,974 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:41,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-04T06:41:41,975 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:41,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742083_1259 (size=1061) 2024-12-04T06:41:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:42,383 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:41:42,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742084_1260 (size=53) 2024-12-04T06:41:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ac3d355fe6fb4092e77435a750f2ad07, disabling compactions & flushes 2024-12-04T06:41:42,789 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. after waiting 0 ms 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:42,789 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:42,789 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac3d355fe6fb4092e77435a750f2ad07: Waiting for close lock at 1733294502789Disabling compacts and flushes for region at 1733294502789Disabling writes for close at 1733294502789Writing region close event to WAL at 1733294502789Closed at 1733294502789 2024-12-04T06:41:42,790 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:41:42,790 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294502790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294502790"}]},"ts":"1733294502790"} 2024-12-04T06:41:42,792 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:41:42,792 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:41:42,792 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294502792"}]},"ts":"1733294502792"} 2024-12-04T06:41:42,794 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:41:42,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, ASSIGN}] 2024-12-04T06:41:42,795 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, ASSIGN 2024-12-04T06:41:42,795 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:41:42,946 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ac3d355fe6fb4092e77435a750f2ad07, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:42,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, ASSIGN because future has completed 2024-12-04T06:41:42,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:41:43,103 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:43,103 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:41:43,103 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,103 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:41:43,103 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,103 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,104 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,105 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:43,106 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac3d355fe6fb4092e77435a750f2ad07 columnFamilyName A 2024-12-04T06:41:43,106 DEBUG [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:43,106 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(327): Store=ac3d355fe6fb4092e77435a750f2ad07/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:43,106 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:43,107 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:43,107 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac3d355fe6fb4092e77435a750f2ad07 columnFamilyName B 2024-12-04T06:41:43,107 DEBUG [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:43,108 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(327): Store=ac3d355fe6fb4092e77435a750f2ad07/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:43,108 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,109 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:41:43,109 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac3d355fe6fb4092e77435a750f2ad07 columnFamilyName C 2024-12-04T06:41:43,109 DEBUG [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:41:43,109 INFO [StoreOpener-ac3d355fe6fb4092e77435a750f2ad07-1 {}] regionserver.HStore(327): Store=ac3d355fe6fb4092e77435a750f2ad07/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:41:43,109 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:43,109 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,110 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,110 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,111 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,111 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,111 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:41:43,112 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,114 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:41:43,114 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened ac3d355fe6fb4092e77435a750f2ad07; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70808750, jitterRate=0.05513259768486023}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:41:43,114 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:43,115 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for ac3d355fe6fb4092e77435a750f2ad07: Running coprocessor pre-open hook at 1733294503103Writing region info on filesystem at 1733294503103Initializing all the Stores at 1733294503104 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294503104Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294503104Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294503104Cleaning up temporary data from old regions at 1733294503111 (+7 ms)Running coprocessor post-open hooks at 1733294503114 (+3 ms)Region opened successfully at 1733294503115 (+1 ms) 2024-12-04T06:41:43,115 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., pid=59, masterSystemTime=1733294503099 2024-12-04T06:41:43,117 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:43,117 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:43,117 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=ac3d355fe6fb4092e77435a750f2ad07, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:43,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:41:43,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T06:41:43,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 in 171 msec 2024-12-04T06:41:43,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-04T06:41:43,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, ASSIGN in 327 msec 2024-12-04T06:41:43,123 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:41:43,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294503123"}]},"ts":"1733294503123"} 2024-12-04T06:41:43,125 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:41:43,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:41:43,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-12-04T06:41:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T06:41:44,127 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:44,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c2d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bacb55b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,130 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149f4777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,131 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,133 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c8bca2f 2024-12-04T06:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:41:44,135 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:41:44,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77284e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1026d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5739e6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6071fcfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,139 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b2cdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ae5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259bf7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,142 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394f22dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62becd7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdb05b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143d209b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,146 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6213ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4a2b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,148 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3201349c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,149 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b8e31a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be8a148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,150 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50272bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,152 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a81884e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7445538a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,153 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6b9c04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eee86c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b9ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,156 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bc44e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605b153a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ce678f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9c3a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f53f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:41:44,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:41:44,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:41:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:41:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:41:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4fc787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:41:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:41:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:41:44,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:41:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fae572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:41:44,164 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,164 DEBUG [Thread-1213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,166 DEBUG [Thread-1219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,166 DEBUG [Thread-1211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,167 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,167 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,174 DEBUG [Thread-1206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,185 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-04T06:41:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:44,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:44,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:44,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:44,189 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:44,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:44,190 DEBUG [Thread-1208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,201 DEBUG [Thread-1215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294564211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294564210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294564214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294564215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294564216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/4f3a5329994744f6babc7e4223f3356d is 50, key is test_row_0/A:col10/1733294504175/Put/seqid=0 2024-12-04T06:41:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,223 DEBUG [Thread-1217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:41:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:41:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:41:44,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:41:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:41:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742085_1261 (size=12001) 2024-12-04T06:41:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:44,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294564326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294564326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294564327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294564327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294564327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:44,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:44,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:44,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:44,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:44,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294564536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294564537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294564537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294564537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294564537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/4f3a5329994744f6babc7e4223f3356d 2024-12-04T06:41:44,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9b34bb0edf0a4007ae041e9a1eba13c3 is 50, key is test_row_0/B:col10/1733294504175/Put/seqid=0 2024-12-04T06:41:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742086_1262 (size=12001) 2024-12-04T06:41:44,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294564846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294564846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294564847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294564847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294564847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:44,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:44,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9b34bb0edf0a4007ae041e9a1eba13c3 2024-12-04T06:41:45,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d385a8df07a945acb87b7b39eaa23435 is 50, key is test_row_0/C:col10/1733294504175/Put/seqid=0 2024-12-04T06:41:45,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742087_1263 (size=12001) 2024-12-04T06:41:45,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:45,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294565357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:45,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294565357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294565357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294565357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294565357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:45,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:45,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d385a8df07a945acb87b7b39eaa23435 2024-12-04T06:41:45,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/4f3a5329994744f6babc7e4223f3356d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d 2024-12-04T06:41:45,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T06:41:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9b34bb0edf0a4007ae041e9a1eba13c3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3 2024-12-04T06:41:45,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T06:41:45,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d385a8df07a945acb87b7b39eaa23435 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435 2024-12-04T06:41:45,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435, entries=150, sequenceid=13, filesize=11.7 K 2024-12-04T06:41:45,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ac3d355fe6fb4092e77435a750f2ad07 in 1357ms, sequenceid=13, compaction requested=false 2024-12-04T06:41:45,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:45,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:45,571 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ffab3ef8d2474910873e5aa54f0fcec3 is 50, key is test_row_0/A:col10/1733294504214/Put/seqid=0 2024-12-04T06:41:45,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742088_1264 (size=12001) 2024-12-04T06:41:45,582 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ffab3ef8d2474910873e5aa54f0fcec3 2024-12-04T06:41:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe2efc1dcbfe45f49ca673dbab8dd54d is 50, key is test_row_0/B:col10/1733294504214/Put/seqid=0 2024-12-04T06:41:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742089_1265 (size=12001) 2024-12-04T06:41:45,595 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe2efc1dcbfe45f49ca673dbab8dd54d 2024-12-04T06:41:45,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6237c13ad66b40248df54b2f4d502e94 is 50, key is test_row_0/C:col10/1733294504214/Put/seqid=0 2024-12-04T06:41:45,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742090_1266 (size=12001) 2024-12-04T06:41:46,008 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6237c13ad66b40248df54b2f4d502e94 2024-12-04T06:41:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ffab3ef8d2474910873e5aa54f0fcec3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3 2024-12-04T06:41:46,017 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:41:46,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe2efc1dcbfe45f49ca673dbab8dd54d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d 2024-12-04T06:41:46,022 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:41:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6237c13ad66b40248df54b2f4d502e94 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94 2024-12-04T06:41:46,031 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:41:46,032 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for ac3d355fe6fb4092e77435a750f2ad07 in 461ms, sequenceid=33, compaction requested=false 2024-12-04T06:41:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-04T06:41:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-04T06:41:46,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T06:41:46,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8440 sec 2024-12-04T06:41:46,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8520 sec 2024-12-04T06:41:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T06:41:46,347 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:46,347 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-04T06:41:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:46,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:46,350 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:46,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:46,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:46,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6983b353a3704d47af0ace147576c255 is 50, key is test_row_0/A:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:46,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742091_1267 (size=14341) 2024-12-04T06:41:46,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6983b353a3704d47af0ace147576c255 2024-12-04T06:41:46,389 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:41:46,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/40ff60bbd30045b3911ef56f2bb00510 is 50, key is test_row_0/B:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742092_1268 (size=12001) 2024-12-04T06:41:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294566424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294566424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294506386, max timestamp=1733294506429Num uniques 859; 2024-12-04T06:41:46,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294506386, max timestamp=1733294506429Num uniques 859; 2024-12-04T06:41:46,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:46,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:46,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294506386, max timestamp=1733294506429Num uniques 859; 2024-12-04T06:41:46,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:46,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294566473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294566473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294566479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294566537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294566537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294566587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294566587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294566587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294566746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294566746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294566797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294566797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294566797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:46,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/40ff60bbd30045b3911ef56f2bb00510 2024-12-04T06:41:46,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/e835f50a1b7d486e94f438da1bcd6d47 is 50, key is test_row_0/C:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:46,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742093_1269 (size=12001) 2024-12-04T06:41:46,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:46,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294567056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294567057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294567106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294567106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294567106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:47,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/e835f50a1b7d486e94f438da1bcd6d47 2024-12-04T06:41:47,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6983b353a3704d47af0ace147576c255 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255 2024-12-04T06:41:47,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255, entries=200, sequenceid=43, filesize=14.0 K 2024-12-04T06:41:47,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/40ff60bbd30045b3911ef56f2bb00510 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510 2024-12-04T06:41:47,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510, entries=150, sequenceid=43, filesize=11.7 K 2024-12-04T06:41:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/e835f50a1b7d486e94f438da1bcd6d47 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47 2024-12-04T06:41:47,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47, entries=150, sequenceid=43, filesize=11.7 K 2024-12-04T06:41:47,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ac3d355fe6fb4092e77435a750f2ad07 in 869ms, sequenceid=43, compaction requested=true 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:47,244 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:47,244 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:47,245 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:47,245 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:47,245 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:47,245 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:47,245 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,245 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,246 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.2 K 2024-12-04T06:41:47,246 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=37.4 K 2024-12-04T06:41:47,246 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b34bb0edf0a4007ae041e9a1eba13c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294504175 2024-12-04T06:41:47,246 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f3a5329994744f6babc7e4223f3356d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294504175 2024-12-04T06:41:47,246 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2efc1dcbfe45f49ca673dbab8dd54d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294504191 2024-12-04T06:41:47,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffab3ef8d2474910873e5aa54f0fcec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294504191 2024-12-04T06:41:47,247 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 40ff60bbd30045b3911ef56f2bb00510, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:47,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6983b353a3704d47af0ace147576c255, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:47,257 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:47,257 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:47,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/1b4e93570f7e4c7092301920f2c61afc is 50, key is test_row_0/B:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:47,258 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/b16b52db237a482cbe9e9d18d0f8e491 is 50, key is test_row_0/A:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742094_1270 (size=12104) 2024-12-04T06:41:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742095_1271 (size=12104) 2024-12-04T06:41:47,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-04T06:41:47,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,272 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:47,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/b59bb0c228fe4eb380ac4dece1d55bc1 is 50, key is test_row_0/A:col10/1733294506460/Put/seqid=0 2024-12-04T06:41:47,284 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/b16b52db237a482cbe9e9d18d0f8e491 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b16b52db237a482cbe9e9d18d0f8e491 2024-12-04T06:41:47,290 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into b16b52db237a482cbe9e9d18d0f8e491(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:47,291 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:47,291 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294507244; duration=0sec 2024-12-04T06:41:47,291 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:47,291 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:47,292 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:47,293 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:47,293 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:47,293 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:47,293 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.2 K 2024-12-04T06:41:47,294 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d385a8df07a945acb87b7b39eaa23435, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733294504175 2024-12-04T06:41:47,294 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6237c13ad66b40248df54b2f4d502e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294504191 2024-12-04T06:41:47,295 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e835f50a1b7d486e94f438da1bcd6d47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742096_1272 (size=12001) 2024-12-04T06:41:47,311 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:47,312 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/09a5184059fb4f15b48c7b6d3cf4c8e2 is 50, key is test_row_0/C:col10/1733294506372/Put/seqid=0 2024-12-04T06:41:47,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742097_1273 (size=12104) 2024-12-04T06:41:47,326 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/09a5184059fb4f15b48c7b6d3cf4c8e2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/09a5184059fb4f15b48c7b6d3cf4c8e2 2024-12-04T06:41:47,332 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into 09a5184059fb4f15b48c7b6d3cf4c8e2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:47,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:47,332 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294507244; duration=0sec 2024-12-04T06:41:47,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:47,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:47,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294567583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294567585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294567616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294567616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294567616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,673 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/1b4e93570f7e4c7092301920f2c61afc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/1b4e93570f7e4c7092301920f2c61afc 2024-12-04T06:41:47,678 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 1b4e93570f7e4c7092301920f2c61afc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:47,678 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:47,678 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294507244; duration=0sec 2024-12-04T06:41:47,678 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:47,678 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294567696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294567696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,705 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/b59bb0c228fe4eb380ac4dece1d55bc1 2024-12-04T06:41:47,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/cd3b93d6a24c4b31b09cb878a98dd02b is 50, key is test_row_0/B:col10/1733294506460/Put/seqid=0 2024-12-04T06:41:47,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742098_1274 (size=12001) 2024-12-04T06:41:47,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294567907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294567907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,122 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/cd3b93d6a24c4b31b09cb878a98dd02b 2024-12-04T06:41:48,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/851cf2e0b6384871a8cfde8fc1b44884 is 50, key is test_row_0/C:col10/1733294506460/Put/seqid=0 2024-12-04T06:41:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742099_1275 (size=12001) 2024-12-04T06:41:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294568216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294568216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:48,535 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/851cf2e0b6384871a8cfde8fc1b44884 2024-12-04T06:41:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/b59bb0c228fe4eb380ac4dece1d55bc1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1 2024-12-04T06:41:48,547 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T06:41:48,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/cd3b93d6a24c4b31b09cb878a98dd02b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b 2024-12-04T06:41:48,551 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T06:41:48,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/851cf2e0b6384871a8cfde8fc1b44884 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884 2024-12-04T06:41:48,555 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884, entries=150, sequenceid=80, filesize=11.7 K 2024-12-04T06:41:48,556 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ac3d355fe6fb4092e77435a750f2ad07 in 1285ms, sequenceid=80, compaction requested=false 2024-12-04T06:41:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-04T06:41:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-04T06:41:48,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-04T06:41:48,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-12-04T06:41:48,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.2120 sec 2024-12-04T06:41:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:48,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:48,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:48,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/9c58e10db03845c4b52ee49bfb624c76 is 50, key is test_row_0/A:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:48,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742100_1276 (size=14341) 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294508634, max timestamp=1733294508663Num uniques 859; 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294508634, max timestamp=1733294508663Num uniques 859; 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294508634, max timestamp=1733294508663Num uniques 859; 2024-12-04T06:41:48,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:41:48,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294568687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294568690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294568693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294568727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294568737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294568796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294568807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294568807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294569006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294569017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294569017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/9c58e10db03845c4b52ee49bfb624c76 2024-12-04T06:41:49,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6dd246dc6b64427490b5dbf019734c94 is 50, key is test_row_0/B:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:49,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742101_1277 (size=12001) 2024-12-04T06:41:49,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6dd246dc6b64427490b5dbf019734c94 2024-12-04T06:41:49,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ebb14b9dbf8a411480a6f8491e413db5 is 50, key is test_row_0/C:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742102_1278 (size=12001) 2024-12-04T06:41:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294569316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294569327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294569327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ebb14b9dbf8a411480a6f8491e413db5 2024-12-04T06:41:49,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/9c58e10db03845c4b52ee49bfb624c76 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76 2024-12-04T06:41:49,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76, entries=200, sequenceid=94, filesize=14.0 K 2024-12-04T06:41:49,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6dd246dc6b64427490b5dbf019734c94 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94 2024-12-04T06:41:49,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94, entries=150, sequenceid=94, filesize=11.7 K 2024-12-04T06:41:49,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ebb14b9dbf8a411480a6f8491e413db5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5 2024-12-04T06:41:49,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5, entries=150, sequenceid=94, filesize=11.7 K 2024-12-04T06:41:49,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ac3d355fe6fb4092e77435a750f2ad07 in 871ms, sequenceid=94, compaction requested=true 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:49,500 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:49,500 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:49,501 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:49,501 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:49,501 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:49,501 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/1b4e93570f7e4c7092301920f2c61afc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.3 K 2024-12-04T06:41:49,502 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:49,502 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:49,502 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:49,502 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b16b52db237a482cbe9e9d18d0f8e491, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=37.5 K 2024-12-04T06:41:49,502 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4e93570f7e4c7092301920f2c61afc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:49,503 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16b52db237a482cbe9e9d18d0f8e491, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:49,503 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3b93d6a24c4b31b09cb878a98dd02b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733294506460 2024-12-04T06:41:49,503 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59bb0c228fe4eb380ac4dece1d55bc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733294506460 2024-12-04T06:41:49,503 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd246dc6b64427490b5dbf019734c94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:49,503 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c58e10db03845c4b52ee49bfb624c76, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:49,512 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:49,512 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:49,513 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/bba15cc23002482f8fb602209ad56e07 is 50, key is test_row_0/B:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:49,513 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/c4770ba59d15427db782fb6154fece7e is 50, key is test_row_0/A:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:49,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742104_1280 (size=12207) 2024-12-04T06:41:49,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742103_1279 (size=12207) 2024-12-04T06:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:49,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-04T06:41:49,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:49,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:49,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/fd57f6354d364b0884f64358adce1585 is 50, key is test_row_0/A:col10/1733294509749/Put/seqid=0 2024-12-04T06:41:49,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742105_1281 (size=14391) 2024-12-04T06:41:49,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294569768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294569773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294569827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294569836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294569846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294569877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294569887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:49,938 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/c4770ba59d15427db782fb6154fece7e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c4770ba59d15427db782fb6154fece7e 2024-12-04T06:41:49,943 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into c4770ba59d15427db782fb6154fece7e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:49,943 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:49,944 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294509500; duration=0sec 2024-12-04T06:41:49,944 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:49,944 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:49,944 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:49,944 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/bba15cc23002482f8fb602209ad56e07 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bba15cc23002482f8fb602209ad56e07 2024-12-04T06:41:49,945 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:49,945 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:49,945 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:49,945 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/09a5184059fb4f15b48c7b6d3cf4c8e2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.3 K 2024-12-04T06:41:49,945 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a5184059fb4f15b48c7b6d3cf4c8e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1733294506369 2024-12-04T06:41:49,946 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851cf2e0b6384871a8cfde8fc1b44884, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733294506460 2024-12-04T06:41:49,946 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb14b9dbf8a411480a6f8491e413db5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:49,952 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into bba15cc23002482f8fb602209ad56e07(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:49,952 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:49,952 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294509500; duration=0sec 2024-12-04T06:41:49,952 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:49,952 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:49,953 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:49,954 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/cb5a4516ffda4885bfde916bde2140f5 is 50, key is test_row_0/C:col10/1733294507578/Put/seqid=0 2024-12-04T06:41:49,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742106_1282 (size=12207) 2024-12-04T06:41:49,972 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/cb5a4516ffda4885bfde916bde2140f5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cb5a4516ffda4885bfde916bde2140f5 2024-12-04T06:41:49,977 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into cb5a4516ffda4885bfde916bde2140f5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:49,977 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:49,977 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294509500; duration=0sec 2024-12-04T06:41:49,977 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:49,977 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294570086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294570097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/fd57f6354d364b0884f64358adce1585 2024-12-04T06:41:50,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85955a16bb1e4c9f914dcfb4cb5d4554 is 50, key is test_row_0/B:col10/1733294509749/Put/seqid=0 2024-12-04T06:41:50,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742107_1283 (size=12051) 2024-12-04T06:41:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294570397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294570406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T06:41:50,517 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:50,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-04T06:41:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T06:41:50,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:50,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:50,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:50,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85955a16bb1e4c9f914dcfb4cb5d4554 2024-12-04T06:41:50,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/bb9114e4ee0641f4bfe307f25b2bf449 is 50, key is test_row_0/C:col10/1733294509749/Put/seqid=0 2024-12-04T06:41:50,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742108_1284 (size=12051) 2024-12-04T06:41:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T06:41:50,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T06:41:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T06:41:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:50,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T06:41:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294570836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,841 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294570847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,851 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294570857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,860 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294570906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294570917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:50,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T06:41:50,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:50,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:50,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/bb9114e4ee0641f4bfe307f25b2bf449 2024-12-04T06:41:51,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/fd57f6354d364b0884f64358adce1585 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585 2024-12-04T06:41:51,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585, entries=200, sequenceid=129, filesize=14.1 K 2024-12-04T06:41:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85955a16bb1e4c9f914dcfb4cb5d4554 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554 2024-12-04T06:41:51,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554, entries=150, sequenceid=129, filesize=11.8 K 2024-12-04T06:41:51,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/bb9114e4ee0641f4bfe307f25b2bf449 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449 2024-12-04T06:41:51,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449, entries=150, sequenceid=129, filesize=11.8 K 2024-12-04T06:41:51,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ac3d355fe6fb4092e77435a750f2ad07 in 1310ms, sequenceid=129, compaction requested=false 2024-12-04T06:41:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:51,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-04T06:41:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:51,133 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:51,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ba46730555814ccdac2b2327c42fdcc2 is 50, key is test_row_0/A:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T06:41:51,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742109_1285 (size=12151) 2024-12-04T06:41:51,187 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ba46730555814ccdac2b2327c42fdcc2 2024-12-04T06:41:51,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/092ab851e0ad41d9988efb40fcdda309 is 50, key is test_row_0/B:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:51,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742110_1286 (size=12151) 2024-12-04T06:41:51,223 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/092ab851e0ad41d9988efb40fcdda309 2024-12-04T06:41:51,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a5e69064120645748ba0e9c4d80cf228 is 50, key is test_row_0/C:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:51,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742111_1287 (size=12151) 2024-12-04T06:41:51,268 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a5e69064120645748ba0e9c4d80cf228 2024-12-04T06:41:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/ba46730555814ccdac2b2327c42fdcc2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2 2024-12-04T06:41:51,284 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2, entries=150, sequenceid=142, filesize=11.9 K 2024-12-04T06:41:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/092ab851e0ad41d9988efb40fcdda309 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309 2024-12-04T06:41:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309, entries=150, sequenceid=142, filesize=11.9 K 2024-12-04T06:41:51,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a5e69064120645748ba0e9c4d80cf228 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228 2024-12-04T06:41:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228, entries=150, sequenceid=142, filesize=11.9 K 2024-12-04T06:41:51,308 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for ac3d355fe6fb4092e77435a750f2ad07 in 175ms, sequenceid=142, compaction requested=true 2024-12-04T06:41:51,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-04T06:41:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-04T06:41:51,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T06:41:51,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 790 msec 2024-12-04T06:41:51,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 795 msec 2024-12-04T06:41:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-04T06:41:51,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:51,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-04T06:41:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T06:41:51,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:51,661 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:51,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T06:41:51,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-04T06:41:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:51,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:51,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-04T06:41:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-04T06:41:51,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-04T06:41:51,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-04T06:41:51,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 160 msec 2024-12-04T06:41:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:51,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:51,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:51,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5dc91cb8804446678323853888010dc9 is 50, key is test_row_1/A:col10/1733294511927/Put/seqid=0 2024-12-04T06:41:51,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742112_1288 (size=12147) 2024-12-04T06:41:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T06:41:51,977 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:51,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294511965, max timestamp=1733294511975Num uniques 859; 2024-12-04T06:41:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294511965, max timestamp=1733294511975Num uniques 859; 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294511965, max timestamp=1733294511975Num uniques 859; 2024-12-04T06:41:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:41:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:51,980 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:51,980 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:51,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:52,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T06:41:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T06:41:52,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5dc91cb8804446678323853888010dc9 2024-12-04T06:41:52,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85927eb7585e4c6f8834a2370012af2b is 50, key is test_row_1/B:col10/1733294511927/Put/seqid=0 2024-12-04T06:41:52,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742113_1289 (size=9757) 2024-12-04T06:41:52,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85927eb7585e4c6f8834a2370012af2b 2024-12-04T06:41:52,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/8fdae2ee689e4aac952c673f78e2d686 is 50, key is test_row_1/C:col10/1733294511927/Put/seqid=0 2024-12-04T06:41:52,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742114_1290 (size=9757) 2024-12-04T06:41:52,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/8fdae2ee689e4aac952c673f78e2d686 2024-12-04T06:41:52,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5dc91cb8804446678323853888010dc9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9 2024-12-04T06:41:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9, entries=150, sequenceid=153, filesize=11.9 K 2024-12-04T06:41:52,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/85927eb7585e4c6f8834a2370012af2b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b 2024-12-04T06:41:52,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b, entries=100, sequenceid=153, filesize=9.5 K 2024-12-04T06:41:52,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/8fdae2ee689e4aac952c673f78e2d686 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686 2024-12-04T06:41:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686, entries=100, sequenceid=153, filesize=9.5 K 2024-12-04T06:41:52,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ac3d355fe6fb4092e77435a750f2ad07 in 466ms, sequenceid=153, compaction requested=true 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:52,396 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:52,396 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:52,399 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:52,399 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:52,399 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,399 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c4770ba59d15427db782fb6154fece7e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=49.7 K 2024-12-04T06:41:52,399 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:52,399 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:52,399 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,399 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bba15cc23002482f8fb602209ad56e07, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=45.1 K 2024-12-04T06:41:52,400 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4770ba59d15427db782fb6154fece7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:52,400 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bba15cc23002482f8fb602209ad56e07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:52,401 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd57f6354d364b0884f64358adce1585, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733294508682 2024-12-04T06:41:52,401 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 85955a16bb1e4c9f914dcfb4cb5d4554, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733294508689 2024-12-04T06:41:52,401 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba46730555814ccdac2b2327c42fdcc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733294509756 2024-12-04T06:41:52,402 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 092ab851e0ad41d9988efb40fcdda309, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733294509756 2024-12-04T06:41:52,402 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dc91cb8804446678323853888010dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294511925 2024-12-04T06:41:52,402 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 85927eb7585e4c6f8834a2370012af2b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294511927 2024-12-04T06:41:52,413 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:52,413 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:52,414 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/3927e1f067944056ad028ae90d02619f is 50, key is test_row_0/A:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:52,414 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/d8e2977bb35f4dc693c8b0e60ec64119 is 50, key is test_row_0/B:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:52,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742115_1291 (size=12493) 2024-12-04T06:41:52,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742116_1292 (size=12493) 2024-12-04T06:41:52,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,439 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:52,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d87170c874e545f784849fdde6779e92 is 50, key is test_row_0/A:col10/1733294512017/Put/seqid=0 2024-12-04T06:41:52,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742117_1293 (size=12151) 2024-12-04T06:41:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:52,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:52,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/d8e2977bb35f4dc693c8b0e60ec64119 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/d8e2977bb35f4dc693c8b0e60ec64119 2024-12-04T06:41:52,830 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/3927e1f067944056ad028ae90d02619f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3927e1f067944056ad028ae90d02619f 2024-12-04T06:41:52,832 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into d8e2977bb35f4dc693c8b0e60ec64119(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:52,832 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:52,832 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=12, startTime=1733294512396; duration=0sec 2024-12-04T06:41:52,832 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:52,832 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:52,832 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:41:52,833 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:41:52,833 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:52,833 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:52,833 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cb5a4516ffda4885bfde916bde2140f5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=45.1 K 2024-12-04T06:41:52,833 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5a4516ffda4885bfde916bde2140f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733294507578 2024-12-04T06:41:52,834 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bb9114e4ee0641f4bfe307f25b2bf449, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733294508689 2024-12-04T06:41:52,835 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e69064120645748ba0e9c4d80cf228, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733294509756 2024-12-04T06:41:52,835 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdae2ee689e4aac952c673f78e2d686, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294511927 2024-12-04T06:41:52,836 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 3927e1f067944056ad028ae90d02619f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:52,836 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:52,836 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=12, startTime=1733294512396; duration=0sec 2024-12-04T06:41:52,837 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:52,837 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:52,843 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:52,844 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/e22c912d985a4e95884dacd5fab31790 is 50, key is test_row_0/C:col10/1733294509772/Put/seqid=0 2024-12-04T06:41:52,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742118_1294 (size=12493) 2024-12-04T06:41:52,851 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d87170c874e545f784849fdde6779e92 2024-12-04T06:41:52,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/7b57fe3f29b443a296bf86e277716dae is 50, key is test_row_0/B:col10/1733294512017/Put/seqid=0 2024-12-04T06:41:52,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294572856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,861 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742119_1295 (size=12151) 2024-12-04T06:41:52,862 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/7b57fe3f29b443a296bf86e277716dae 2024-12-04T06:41:52,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a4d9ad19e259478c866feae0bfe0fc06 is 50, key is test_row_0/C:col10/1733294512017/Put/seqid=0 2024-12-04T06:41:52,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294572867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,871 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742120_1296 (size=12151) 2024-12-04T06:41:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294572877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,879 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294572987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294572997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:53,252 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/e22c912d985a4e95884dacd5fab31790 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e22c912d985a4e95884dacd5fab31790 2024-12-04T06:41:53,256 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into e22c912d985a4e95884dacd5fab31790(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:53,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:53,257 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=12, startTime=1733294512396; duration=0sec 2024-12-04T06:41:53,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:53,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:53,279 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a4d9ad19e259478c866feae0bfe0fc06 2024-12-04T06:41:53,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d87170c874e545f784849fdde6779e92 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92 2024-12-04T06:41:53,288 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T06:41:53,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/7b57fe3f29b443a296bf86e277716dae as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae 2024-12-04T06:41:53,292 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T06:41:53,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a4d9ad19e259478c866feae0bfe0fc06 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06 2024-12-04T06:41:53,297 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06, entries=150, sequenceid=190, filesize=11.9 K 2024-12-04T06:41:53,298 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ac3d355fe6fb4092e77435a750f2ad07 in 859ms, sequenceid=190, compaction requested=false 2024-12-04T06:41:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:53,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-04T06:41:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-04T06:41:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:53,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:53,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T06:41:53,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-12-04T06:41:53,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3250 sec 2024-12-04T06:41:53,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5b29f289965349939b149a842e208aaa is 50, key is test_row_0/A:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:53,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742121_1297 (size=14541) 2024-12-04T06:41:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294513326, max timestamp=1733294513347Num uniques 859; 2024-12-04T06:41:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294513326, max timestamp=1733294513347Num uniques 859; 2024-12-04T06:41:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294513326, max timestamp=1733294513347Num uniques 859; 2024-12-04T06:41:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294573423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294573423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294573537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294573537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5b29f289965349939b149a842e208aaa 2024-12-04T06:41:53,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/bcac7db8776347898599cabaf30a8b4c is 50, key is test_row_0/B:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:53,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742122_1298 (size=12151) 2024-12-04T06:41:53,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294573747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294573747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:53,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294574057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294574057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T06:41:54,127 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:54,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-04T06:41:54,130 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T06:41:54,130 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:54,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:54,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/bcac7db8776347898599cabaf30a8b4c 2024-12-04T06:41:54,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ffc556ac55fa43a0a2a2227ede0ba971 is 50, key is test_row_0/C:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:54,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742123_1299 (size=12151) 2024-12-04T06:41:54,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ffc556ac55fa43a0a2a2227ede0ba971 2024-12-04T06:41:54,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/5b29f289965349939b149a842e208aaa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa 2024-12-04T06:41:54,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa, entries=200, sequenceid=204, filesize=14.2 K 2024-12-04T06:41:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/bcac7db8776347898599cabaf30a8b4c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c 2024-12-04T06:41:54,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c, entries=150, sequenceid=204, filesize=11.9 K 2024-12-04T06:41:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/ffc556ac55fa43a0a2a2227ede0ba971 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971 2024-12-04T06:41:54,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971, entries=150, sequenceid=204, filesize=11.9 K 2024-12-04T06:41:54,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ac3d355fe6fb4092e77435a750f2ad07 in 871ms, sequenceid=204, compaction requested=true 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:54,171 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:54,171 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:54,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:54,172 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:54,172 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:54,172 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:54,172 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:54,172 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:54,172 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:54,172 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3927e1f067944056ad028ae90d02619f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=38.3 K 2024-12-04T06:41:54,172 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/d8e2977bb35f4dc693c8b0e60ec64119, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.9 K 2024-12-04T06:41:54,174 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3927e1f067944056ad028ae90d02619f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294509772 2024-12-04T06:41:54,174 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e2977bb35f4dc693c8b0e60ec64119, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294509772 2024-12-04T06:41:54,174 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d87170c874e545f784849fdde6779e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294512010 2024-12-04T06:41:54,174 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b57fe3f29b443a296bf86e277716dae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294512010 2024-12-04T06:41:54,175 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bcac7db8776347898599cabaf30a8b4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:54,175 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b29f289965349939b149a842e208aaa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:54,184 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:54,185 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/2bd39c8c6c4f44af898d5ac3ba6a07dc is 50, key is test_row_0/A:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:54,187 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:54,188 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/8798d64ff1884a489b3e83af566b0898 is 50, key is test_row_0/B:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:54,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742124_1300 (size=12595) 2024-12-04T06:41:54,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742125_1301 (size=12595) 2024-12-04T06:41:54,204 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/2bd39c8c6c4f44af898d5ac3ba6a07dc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/2bd39c8c6c4f44af898d5ac3ba6a07dc 2024-12-04T06:41:54,209 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 2bd39c8c6c4f44af898d5ac3ba6a07dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:54,209 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:54,209 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294514171; duration=0sec 2024-12-04T06:41:54,209 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:54,209 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:54,209 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:54,210 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:54,210 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:54,210 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:54,210 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e22c912d985a4e95884dacd5fab31790, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=35.9 K 2024-12-04T06:41:54,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22c912d985a4e95884dacd5fab31790, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733294509772 2024-12-04T06:41:54,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d9ad19e259478c866feae0bfe0fc06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1733294512010 2024-12-04T06:41:54,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc556ac55fa43a0a2a2227ede0ba971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:54,221 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:54,222 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/3c81816a2b0a4dc3919a1b74db665acc is 50, key is test_row_0/C:col10/1733294512672/Put/seqid=0 2024-12-04T06:41:54,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742126_1302 (size=12595) 2024-12-04T06:41:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T06:41:54,245 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/3c81816a2b0a4dc3919a1b74db665acc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3c81816a2b0a4dc3919a1b74db665acc 2024-12-04T06:41:54,251 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into 3c81816a2b0a4dc3919a1b74db665acc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:54,251 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:54,251 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294514171; duration=0sec 2024-12-04T06:41:54,251 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:54,251 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:54,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-04T06:41:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:54,283 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:54,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:41:54,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/40866ef697d245c1986790eecfd0c3fa is 50, key is test_row_0/A:col10/1733294513394/Put/seqid=0 2024-12-04T06:41:54,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742127_1303 (size=12151) 2024-12-04T06:41:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T06:41:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:54,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:54,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294574590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294574589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,607 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/8798d64ff1884a489b3e83af566b0898 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/8798d64ff1884a489b3e83af566b0898 2024-12-04T06:41:54,612 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 8798d64ff1884a489b3e83af566b0898(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:54,612 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:54,612 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294514171; duration=0sec 2024-12-04T06:41:54,612 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:54,612 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:54,693 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/40866ef697d245c1986790eecfd0c3fa 2024-12-04T06:41:54,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c28f269691a74757a94b7cfce4608cad is 50, key is test_row_0/B:col10/1733294513394/Put/seqid=0 2024-12-04T06:41:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742128_1304 (size=12151) 2024-12-04T06:41:54,705 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c28f269691a74757a94b7cfce4608cad 2024-12-04T06:41:54,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294574706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294574706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/33bab05fc4f14038835eb945f44d87b3 is 50, key is test_row_0/C:col10/1733294513394/Put/seqid=0 2024-12-04T06:41:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742129_1305 (size=12151) 2024-12-04T06:41:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T06:41:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294574916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294574916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,117 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/33bab05fc4f14038835eb945f44d87b3 2024-12-04T06:41:55,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/40866ef697d245c1986790eecfd0c3fa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa 2024-12-04T06:41:55,125 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T06:41:55,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c28f269691a74757a94b7cfce4608cad as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad 2024-12-04T06:41:55,129 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T06:41:55,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/33bab05fc4f14038835eb945f44d87b3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3 2024-12-04T06:41:55,132 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3, entries=150, sequenceid=241, filesize=11.9 K 2024-12-04T06:41:55,133 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ac3d355fe6fb4092e77435a750f2ad07 in 850ms, sequenceid=241, compaction requested=false 2024-12-04T06:41:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-04T06:41:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-04T06:41:55,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-04T06:41:55,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-12-04T06:41:55,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0090 sec 2024-12-04T06:41:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:55,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:41:55,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:55,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:55,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/11e569d9f7284b24af664693e2e385ce is 50, key is test_row_0/A:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:55,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742130_1306 (size=12151) 2024-12-04T06:41:55,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/11e569d9f7284b24af664693e2e385ce 2024-12-04T06:41:55,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9e6061765a194c9dbddb231e194e3d5b is 50, key is test_row_0/B:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742131_1307 (size=12151) 2024-12-04T06:41:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-04T06:41:55,267 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:55,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-04T06:41:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:55,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:55,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:55,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:55,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294575278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294575278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294575386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294575386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:55,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294575596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294575596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9e6061765a194c9dbddb231e194e3d5b 2024-12-04T06:41:55,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6ac655ff96bf44758e9e3869c75436f8 is 50, key is test_row_0/C:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742132_1308 (size=12151) 2024-12-04T06:41:55,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:55,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:55,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294575907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294575907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:56,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:56,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6ac655ff96bf44758e9e3869c75436f8 2024-12-04T06:41:56,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/11e569d9f7284b24af664693e2e385ce as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce 2024-12-04T06:41:56,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce, entries=150, sequenceid=254, filesize=11.9 K 2024-12-04T06:41:56,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/9e6061765a194c9dbddb231e194e3d5b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b 2024-12-04T06:41:56,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b, entries=150, sequenceid=254, filesize=11.9 K 2024-12-04T06:41:56,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/6ac655ff96bf44758e9e3869c75436f8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8 2024-12-04T06:41:56,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8, entries=150, sequenceid=254, filesize=11.9 K 2024-12-04T06:41:56,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ac3d355fe6fb4092e77435a750f2ad07 in 858ms, sequenceid=254, compaction requested=true 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:56,087 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:56,087 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:56,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:56,088 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:56,088 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:56,088 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:56,088 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:56,088 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,088 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,088 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/2bd39c8c6c4f44af898d5ac3ba6a07dc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.0 K 2024-12-04T06:41:56,088 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/8798d64ff1884a489b3e83af566b0898, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.0 K 2024-12-04T06:41:56,088 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd39c8c6c4f44af898d5ac3ba6a07dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:56,089 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8798d64ff1884a489b3e83af566b0898, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:56,089 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40866ef697d245c1986790eecfd0c3fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733294513394 2024-12-04T06:41:56,089 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c28f269691a74757a94b7cfce4608cad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733294513394 2024-12-04T06:41:56,089 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11e569d9f7284b24af664693e2e385ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:56,089 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e6061765a194c9dbddb231e194e3d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:56,096 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:56,096 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/941fa3d28bba4defbb64a57a180469a3 is 50, key is test_row_0/B:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:56,099 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:56,100 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/72592a9c687f4df3b0bbe867628b6409 is 50, key is test_row_0/A:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742133_1309 (size=12697) 2024-12-04T06:41:56,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742134_1310 (size=12697) 2024-12-04T06:41:56,113 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/941fa3d28bba4defbb64a57a180469a3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/941fa3d28bba4defbb64a57a180469a3 2024-12-04T06:41:56,118 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 941fa3d28bba4defbb64a57a180469a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:56,118 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:56,118 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294516087; duration=0sec 2024-12-04T06:41:56,118 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:56,118 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:56,118 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:56,119 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:56,119 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:56,119 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,119 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3c81816a2b0a4dc3919a1b74db665acc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.0 K 2024-12-04T06:41:56,120 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c81816a2b0a4dc3919a1b74db665acc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733294512666 2024-12-04T06:41:56,120 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 33bab05fc4f14038835eb945f44d87b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733294513394 2024-12-04T06:41:56,120 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac655ff96bf44758e9e3869c75436f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:56,127 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:56,128 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/c97123072ae149dba8bb647b94a31aa1 is 50, key is test_row_0/C:col10/1733294514581/Put/seqid=0 2024-12-04T06:41:56,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742135_1311 (size=12697) 2024-12-04T06:41:56,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-04T06:41:56,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:56,189 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:41:56,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:56,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/c335a3f8e43a4d49b9626c70b589d979 is 50, key is test_row_0/A:col10/1733294515277/Put/seqid=0 2024-12-04T06:41:56,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742136_1312 (size=12301) 2024-12-04T06:41:56,202 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/c335a3f8e43a4d49b9626c70b589d979 2024-12-04T06:41:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/26e42ccf91914d8a98f035f446403023 is 50, key is test_row_0/B:col10/1733294515277/Put/seqid=0 2024-12-04T06:41:56,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742137_1313 (size=12301) 2024-12-04T06:41:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:56,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294576435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294576436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,514 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/72592a9c687f4df3b0bbe867628b6409 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/72592a9c687f4df3b0bbe867628b6409 2024-12-04T06:41:56,518 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 72592a9c687f4df3b0bbe867628b6409(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:56,518 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:56,518 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294516087; duration=0sec 2024-12-04T06:41:56,519 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:56,519 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:56,537 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/c97123072ae149dba8bb647b94a31aa1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c97123072ae149dba8bb647b94a31aa1 2024-12-04T06:41:56,541 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into c97123072ae149dba8bb647b94a31aa1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:56,541 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:56,542 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294516087; duration=0sec 2024-12-04T06:41:56,542 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:56,542 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294576547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294576547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,613 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/26e42ccf91914d8a98f035f446403023 2024-12-04T06:41:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/5287d8bdcbf84430bd78bd05b222a1e4 is 50, key is test_row_0/C:col10/1733294515277/Put/seqid=0 2024-12-04T06:41:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742138_1314 (size=12301) 2024-12-04T06:41:56,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294576756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294576757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42352 deadline: 1733294576887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,889 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42300 deadline: 1733294576897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,899 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42314 deadline: 1733294576906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:56,910 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/5287d8bdcbf84430bd78bd05b222a1e4 2024-12-04T06:41:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/c335a3f8e43a4d49b9626c70b589d979 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979 2024-12-04T06:41:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979, entries=150, sequenceid=275, filesize=12.0 K 2024-12-04T06:41:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/26e42ccf91914d8a98f035f446403023 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023 2024-12-04T06:41:57,039 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023, entries=150, sequenceid=275, filesize=12.0 K 2024-12-04T06:41:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/5287d8bdcbf84430bd78bd05b222a1e4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4 2024-12-04T06:41:57,043 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4, entries=150, sequenceid=275, filesize=12.0 K 2024-12-04T06:41:57,044 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ac3d355fe6fb4092e77435a750f2ad07 in 855ms, sequenceid=275, compaction requested=false 2024-12-04T06:41:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-04T06:41:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-04T06:41:57,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-04T06:41:57,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-12-04T06:41:57,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7820 sec 2024-12-04T06:41:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:57,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:57,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/66d41650c3624ff58979c7d65767e4a4 is 50, key is test_row_0/A:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:57,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742139_1315 (size=12297) 2024-12-04T06:41:57,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294577100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294577102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294577207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294577217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-04T06:41:57,417 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:57,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-04T06:41:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294577417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,420 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:57,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294577427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/66d41650c3624ff58979c7d65767e4a4 2024-12-04T06:41:57,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6fac4d5663b34be99a45ef76bd8ae1e9 is 50, key is test_row_0/B:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:57,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742140_1316 (size=9857) 2024-12-04T06:41:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:57,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:57,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294577726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294577736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:57,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:57,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:57,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6fac4d5663b34be99a45ef76bd8ae1e9 2024-12-04T06:41:57,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/676dfbad9b7947daa6bb63c2ea9ccebe is 50, key is test_row_0/C:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742141_1317 (size=9857) 2024-12-04T06:41:58,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:58,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:58,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294578236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294578247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:58,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/676dfbad9b7947daa6bb63c2ea9ccebe 2024-12-04T06:41:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/66d41650c3624ff58979c7d65767e4a4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4 2024-12-04T06:41:58,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4, entries=150, sequenceid=291, filesize=12.0 K 2024-12-04T06:41:58,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6fac4d5663b34be99a45ef76bd8ae1e9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9 2024-12-04T06:41:58,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9, entries=100, sequenceid=291, filesize=9.6 K 2024-12-04T06:41:58,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/676dfbad9b7947daa6bb63c2ea9ccebe as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe 2024-12-04T06:41:58,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe, entries=100, sequenceid=291, filesize=9.6 K 2024-12-04T06:41:58,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ac3d355fe6fb4092e77435a750f2ad07 in 1262ms, sequenceid=291, compaction requested=true 2024-12-04T06:41:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:41:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:58,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:41:58,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:58,331 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:58,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:41:58,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:58,331 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:58,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:58,332 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:58,332 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:41:58,332 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:41:58,332 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,332 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,332 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/941fa3d28bba4defbb64a57a180469a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=34.0 K 2024-12-04T06:41:58,332 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/72592a9c687f4df3b0bbe867628b6409, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.4 K 2024-12-04T06:41:58,332 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 941fa3d28bba4defbb64a57a180469a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:58,333 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72592a9c687f4df3b0bbe867628b6409, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:58,333 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e42ccf91914d8a98f035f446403023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733294515270 2024-12-04T06:41:58,333 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c335a3f8e43a4d49b9626c70b589d979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733294515270 2024-12-04T06:41:58,333 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fac4d5663b34be99a45ef76bd8ae1e9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294516435 2024-12-04T06:41:58,333 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d41650c3624ff58979c7d65767e4a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294516435 2024-12-04T06:41:58,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-04T06:41:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,339 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:58,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:58,347 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:58,347 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d85da137ad3b4995912df90070aefd9c is 50, key is test_row_0/A:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:58,350 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:58,351 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/dc3bc1d9fd184a6ea39fd88c9d01f558 is 50, key is test_row_0/B:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:58,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6df19fb1f40d42ffa2a9684ab4145ed0 is 50, key is test_row_0/A:col10/1733294517083/Put/seqid=0 2024-12-04T06:41:58,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742142_1318 (size=12949) 2024-12-04T06:41:58,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742143_1319 (size=12949) 2024-12-04T06:41:58,370 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/dc3bc1d9fd184a6ea39fd88c9d01f558 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/dc3bc1d9fd184a6ea39fd88c9d01f558 2024-12-04T06:41:58,375 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into dc3bc1d9fd184a6ea39fd88c9d01f558(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:58,375 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:58,375 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294518330; duration=0sec 2024-12-04T06:41:58,375 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:41:58,375 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:41:58,375 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:41:58,377 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:41:58,377 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:41:58,378 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,378 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c97123072ae149dba8bb647b94a31aa1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=34.0 K 2024-12-04T06:41:58,378 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c97123072ae149dba8bb647b94a31aa1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733294514581 2024-12-04T06:41:58,378 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5287d8bdcbf84430bd78bd05b222a1e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733294515270 2024-12-04T06:41:58,379 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 676dfbad9b7947daa6bb63c2ea9ccebe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294516435 2024-12-04T06:41:58,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742144_1320 (size=12301) 2024-12-04T06:41:58,386 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:41:58,387 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/2ce8da0b537f45f49503534311994de9 is 50, key is test_row_0/C:col10/1733294516435/Put/seqid=0 2024-12-04T06:41:58,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742145_1321 (size=12949) 2024-12-04T06:41:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:58,770 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d85da137ad3b4995912df90070aefd9c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d85da137ad3b4995912df90070aefd9c 2024-12-04T06:41:58,775 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into d85da137ad3b4995912df90070aefd9c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:58,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:58,775 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294518330; duration=0sec 2024-12-04T06:41:58,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:58,775 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:41:58,784 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6df19fb1f40d42ffa2a9684ab4145ed0 2024-12-04T06:41:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/3f9114ab46a54fde8013facd0c08ed7f is 50, key is test_row_0/B:col10/1733294517083/Put/seqid=0 2024-12-04T06:41:58,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742146_1322 (size=12301) 2024-12-04T06:41:58,795 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/3f9114ab46a54fde8013facd0c08ed7f 2024-12-04T06:41:58,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/b08fbca940044b6da5d36511dfeea5b6 is 50, key is test_row_0/C:col10/1733294517083/Put/seqid=0 2024-12-04T06:41:58,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742147_1323 (size=12301) 2024-12-04T06:41:58,812 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/b08fbca940044b6da5d36511dfeea5b6 2024-12-04T06:41:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6df19fb1f40d42ffa2a9684ab4145ed0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0 2024-12-04T06:41:58,822 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/2ce8da0b537f45f49503534311994de9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/2ce8da0b537f45f49503534311994de9 2024-12-04T06:41:58,826 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0, entries=150, sequenceid=309, filesize=12.0 K 2024-12-04T06:41:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/3f9114ab46a54fde8013facd0c08ed7f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f 2024-12-04T06:41:58,829 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into 2ce8da0b537f45f49503534311994de9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:41:58,829 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:58,829 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294518331; duration=0sec 2024-12-04T06:41:58,829 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:41:58,829 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:41:58,833 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f, entries=150, sequenceid=309, filesize=12.0 K 2024-12-04T06:41:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/b08fbca940044b6da5d36511dfeea5b6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6 2024-12-04T06:41:58,839 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6, entries=150, sequenceid=309, filesize=12.0 K 2024-12-04T06:41:58,840 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ac3d355fe6fb4092e77435a750f2ad07 in 502ms, sequenceid=309, compaction requested=false 2024-12-04T06:41:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:41:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:58,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-04T06:41:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-04T06:41:58,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-04T06:41:58,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-12-04T06:41:58,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4270 sec 2024-12-04T06:41:59,081 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:41:59,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:41:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:41:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:41:59,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/0a78568323a1433d979c9f6748b12e2b is 50, key is test_row_0/A:col10/1733294519265/Put/seqid=0 2024-12-04T06:41:59,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742148_1324 (size=12301) 2024-12-04T06:41:59,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294519310, max timestamp=1733294519334Num uniques 859; 2024-12-04T06:41:59,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:59,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294519310, max timestamp=1733294519334Num uniques 859; 2024-12-04T06:41:59,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:59,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294519310, max timestamp=1733294519334Num uniques 859; 2024-12-04T06:41:59,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:41:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294579404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294579402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294579516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294579517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-04T06:41:59,568 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:41:59,568 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:41:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-04T06:41:59,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:41:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:41:59,572 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:41:59,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:41:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:41:59,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/0a78568323a1433d979c9f6748b12e2b 2024-12-04T06:41:59,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T06:41:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:59,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c9272659d6514f46adce8f53b6165f10 is 50, key is test_row_0/B:col10/1733294519265/Put/seqid=0 2024-12-04T06:41:59,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294579727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:41:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294579727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:41:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:41:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:41:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742149_1325 (size=12301) 2024-12-04T06:41:59,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c9272659d6514f46adce8f53b6165f10 2024-12-04T06:41:59,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f59735b559ee4aa7a232dbed2871e35c is 50, key is test_row_0/C:col10/1733294519265/Put/seqid=0 2024-12-04T06:41:59,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742150_1326 (size=12301) 2024-12-04T06:41:59,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T06:41:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:41:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:41:59,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:41:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:42:00,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T06:42:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:00,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:00,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294580037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294580037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f59735b559ee4aa7a232dbed2871e35c 2024-12-04T06:42:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/0a78568323a1433d979c9f6748b12e2b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b 2024-12-04T06:42:00,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b, entries=150, sequenceid=322, filesize=12.0 K 2024-12-04T06:42:00,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c9272659d6514f46adce8f53b6165f10 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10 2024-12-04T06:42:00,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10, entries=150, sequenceid=322, filesize=12.0 K 2024-12-04T06:42:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f59735b559ee4aa7a232dbed2871e35c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c 2024-12-04T06:42:00,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c, entries=150, sequenceid=322, filesize=12.0 K 2024-12-04T06:42:00,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ac3d355fe6fb4092e77435a750f2ad07 in 918ms, sequenceid=322, compaction requested=true 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:00,187 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:00,187 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:00,188 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:00,188 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:42:00,188 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,188 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:00,189 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:42:00,189 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d85da137ad3b4995912df90070aefd9c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.7 K 2024-12-04T06:42:00,189 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,189 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/dc3bc1d9fd184a6ea39fd88c9d01f558, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.7 K 2024-12-04T06:42:00,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,190 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:00,191 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d85da137ad3b4995912df90070aefd9c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294515276 2024-12-04T06:42:00,191 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting dc3bc1d9fd184a6ea39fd88c9d01f558, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294515276 2024-12-04T06:42:00,191 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df19fb1f40d42ffa2a9684ab4145ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733294517083 2024-12-04T06:42:00,191 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f9114ab46a54fde8013facd0c08ed7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733294517083 2024-12-04T06:42:00,192 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a78568323a1433d979c9f6748b12e2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:00,192 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c9272659d6514f46adce8f53b6165f10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:42:00,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d3141cc2c2374fd59b563627eec9e8bf is 50, key is test_row_0/A:col10/1733294519402/Put/seqid=0 2024-12-04T06:42:00,202 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:00,202 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:00,202 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/655e102e3f674886816996969ff48b57 is 50, key is test_row_0/A:col10/1733294519265/Put/seqid=0 2024-12-04T06:42:00,204 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/5d1ec6cd12844eca81716bae6617e5c7 is 50, key is test_row_0/B:col10/1733294519265/Put/seqid=0 2024-12-04T06:42:00,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742152_1328 (size=13051) 2024-12-04T06:42:00,228 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/5d1ec6cd12844eca81716bae6617e5c7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/5d1ec6cd12844eca81716bae6617e5c7 2024-12-04T06:42:00,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742151_1327 (size=13051) 2024-12-04T06:42:00,235 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 5d1ec6cd12844eca81716bae6617e5c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:00,235 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:00,236 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294520187; duration=0sec 2024-12-04T06:42:00,236 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/655e102e3f674886816996969ff48b57 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/655e102e3f674886816996969ff48b57 2024-12-04T06:42:00,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:00,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:42:00,236 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:00,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:00,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:42:00,238 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,238 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/2ce8da0b537f45f49503534311994de9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.7 K 2024-12-04T06:42:00,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce8da0b537f45f49503534311994de9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733294515276 2024-12-04T06:42:00,239 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b08fbca940044b6da5d36511dfeea5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733294517083 2024-12-04T06:42:00,240 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f59735b559ee4aa7a232dbed2871e35c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:00,243 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 655e102e3f674886816996969ff48b57(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:00,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:00,243 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294520187; duration=0sec 2024-12-04T06:42:00,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:00,243 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:42:00,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742153_1329 (size=12301) 2024-12-04T06:42:00,247 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d3141cc2c2374fd59b563627eec9e8bf 2024-12-04T06:42:00,256 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:00,257 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a4df28b0e62e4ff784243b697005795f is 50, key is test_row_0/C:col10/1733294519265/Put/seqid=0 2024-12-04T06:42:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/007fae864b6a4f1bb0b425a37ec8de15 is 50, key is test_row_0/B:col10/1733294519402/Put/seqid=0 2024-12-04T06:42:00,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742154_1330 (size=13051) 2024-12-04T06:42:00,284 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/a4df28b0e62e4ff784243b697005795f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4df28b0e62e4ff784243b697005795f 2024-12-04T06:42:00,291 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into a4df28b0e62e4ff784243b697005795f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:00,291 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:00,291 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294520187; duration=0sec 2024-12-04T06:42:00,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742155_1331 (size=12301) 2024-12-04T06:42:00,291 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:00,292 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:42:00,293 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/007fae864b6a4f1bb0b425a37ec8de15 2024-12-04T06:42:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d45e77153e8d467098627cbeb418f854 is 50, key is test_row_0/C:col10/1733294519402/Put/seqid=0 2024-12-04T06:42:00,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742156_1332 (size=12301) 2024-12-04T06:42:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:00,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:00,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294580584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294580591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294580697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:42:00,711 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d45e77153e8d467098627cbeb418f854 2024-12-04T06:42:00,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294580707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:00,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/d3141cc2c2374fd59b563627eec9e8bf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf 2024-12-04T06:42:00,721 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf, entries=150, sequenceid=358, filesize=12.0 K 2024-12-04T06:42:00,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/007fae864b6a4f1bb0b425a37ec8de15 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15 2024-12-04T06:42:00,727 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15, entries=150, sequenceid=358, filesize=12.0 K 2024-12-04T06:42:00,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d45e77153e8d467098627cbeb418f854 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854 2024-12-04T06:42:00,735 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854, entries=150, sequenceid=358, filesize=12.0 K 2024-12-04T06:42:00,736 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ac3d355fe6fb4092e77435a750f2ad07 in 546ms, sequenceid=358, compaction requested=false 2024-12-04T06:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-04T06:42:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-04T06:42:00,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-04T06:42:00,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-12-04T06:42:00,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1740 sec 2024-12-04T06:42:00,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:00,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:00,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/db22e06ed32449ebba2803017c26168c is 50, key is test_row_0/A:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:00,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742157_1333 (size=14741) 2024-12-04T06:42:00,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/db22e06ed32449ebba2803017c26168c 2024-12-04T06:42:00,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6470dbd7d30341828b4a7da0f976337d is 50, key is test_row_0/B:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:01,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742158_1334 (size=12301) 2024-12-04T06:42:01,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6470dbd7d30341828b4a7da0f976337d 2024-12-04T06:42:01,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f46a299217414c039fd36a6b5f564673 is 50, key is test_row_0/C:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:01,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294521004, max timestamp=1733294521053Num uniques 859; 2024-12-04T06:42:01,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294521004, max timestamp=1733294521053Num uniques 859; 2024-12-04T06:42:01,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:01,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:01,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294521004, max timestamp=1733294521053Num uniques 859; 2024-12-04T06:42:01,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742159_1335 (size=12301) 2024-12-04T06:42:01,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f46a299217414c039fd36a6b5f564673 2024-12-04T06:42:01,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/db22e06ed32449ebba2803017c26168c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c 2024-12-04T06:42:01,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c, entries=200, sequenceid=372, filesize=14.4 K 2024-12-04T06:42:01,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/6470dbd7d30341828b4a7da0f976337d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d 2024-12-04T06:42:01,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d, entries=150, sequenceid=372, filesize=12.0 K 2024-12-04T06:42:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/f46a299217414c039fd36a6b5f564673 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673 2024-12-04T06:42:01,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673, entries=150, sequenceid=372, filesize=12.0 K 2024-12-04T06:42:01,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for ac3d355fe6fb4092e77435a750f2ad07 in 212ms, sequenceid=372, compaction requested=true 2024-12-04T06:42:01,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:01,126 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:01,126 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:01,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:01,127 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:01,127 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:01,127 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:42:01,127 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:42:01,127 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:01,127 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:01,127 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/655e102e3f674886816996969ff48b57, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=39.2 K 2024-12-04T06:42:01,127 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/5d1ec6cd12844eca81716bae6617e5c7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.8 K 2024-12-04T06:42:01,128 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1ec6cd12844eca81716bae6617e5c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:01,128 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655e102e3f674886816996969ff48b57, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:01,128 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 007fae864b6a4f1bb0b425a37ec8de15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733294519384 2024-12-04T06:42:01,129 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3141cc2c2374fd59b563627eec9e8bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733294519384 2024-12-04T06:42:01,129 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6470dbd7d30341828b4a7da0f976337d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:01,129 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting db22e06ed32449ebba2803017c26168c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:01,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:01,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:01,141 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:01,142 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/14a605e796604a509d2b6362c430e83b is 50, key is test_row_0/B:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:01,151 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:01,152 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6cb0b2fcdd984149a761e562bbee0ca4 is 50, key is test_row_0/A:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:01,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/80a9c2b2235a4467acb130cc1d931328 is 50, key is test_row_0/A:col10/1733294521135/Put/seqid=0 2024-12-04T06:42:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294581180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294581189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742160_1336 (size=13153) 2024-12-04T06:42:01,212 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/14a605e796604a509d2b6362c430e83b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/14a605e796604a509d2b6362c430e83b 2024-12-04T06:42:01,218 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 14a605e796604a509d2b6362c430e83b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:01,218 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:01,218 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294521126; duration=0sec 2024-12-04T06:42:01,219 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:01,219 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:42:01,219 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:01,220 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:01,220 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:42:01,220 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:01,221 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4df28b0e62e4ff784243b697005795f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.8 K 2024-12-04T06:42:01,221 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a4df28b0e62e4ff784243b697005795f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733294519257 2024-12-04T06:42:01,221 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d45e77153e8d467098627cbeb418f854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733294519384 2024-12-04T06:42:01,222 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f46a299217414c039fd36a6b5f564673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:01,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742161_1337 (size=13153) 2024-12-04T06:42:01,255 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/6cb0b2fcdd984149a761e562bbee0ca4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6cb0b2fcdd984149a761e562bbee0ca4 2024-12-04T06:42:01,262 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 6cb0b2fcdd984149a761e562bbee0ca4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:01,262 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:01,262 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294521125; duration=0sec 2024-12-04T06:42:01,262 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:01,262 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:42:01,273 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:01,273 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d26479367b66402abadeb4a0e7c95edc is 50, key is test_row_0/C:col10/1733294520910/Put/seqid=0 2024-12-04T06:42:01,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742162_1338 (size=17181) 2024-12-04T06:42:01,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/80a9c2b2235a4467acb130cc1d931328 2024-12-04T06:42:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294581297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294581307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/31654f34d02c46ca9e123ba952e57de3 is 50, key is test_row_0/B:col10/1733294521135/Put/seqid=0 2024-12-04T06:42:01,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742163_1339 (size=13153) 2024-12-04T06:42:01,331 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d26479367b66402abadeb4a0e7c95edc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d26479367b66402abadeb4a0e7c95edc 2024-12-04T06:42:01,338 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into d26479367b66402abadeb4a0e7c95edc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:01,338 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:01,338 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294521126; duration=0sec 2024-12-04T06:42:01,339 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:01,339 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:42:01,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742164_1340 (size=12301) 2024-12-04T06:42:01,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294581508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294581517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-04T06:42:01,717 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:01,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-04T06:42:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:01,720 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:01,721 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:01,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:01,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/31654f34d02c46ca9e123ba952e57de3 2024-12-04T06:42:01,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/61c2fb0544a4415491c111cdf358714e is 50, key is test_row_0/C:col10/1733294521135/Put/seqid=0 2024-12-04T06:42:01,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742165_1341 (size=12301) 2024-12-04T06:42:01,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/61c2fb0544a4415491c111cdf358714e 2024-12-04T06:42:01,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/80a9c2b2235a4467acb130cc1d931328 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328 2024-12-04T06:42:01,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328, entries=250, sequenceid=408, filesize=16.8 K 2024-12-04T06:42:01,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/31654f34d02c46ca9e123ba952e57de3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3 2024-12-04T06:42:01,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3, entries=150, sequenceid=408, filesize=12.0 K 2024-12-04T06:42:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/61c2fb0544a4415491c111cdf358714e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e 2024-12-04T06:42:01,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e, entries=150, sequenceid=408, filesize=12.0 K 2024-12-04T06:42:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294581817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:01,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for ac3d355fe6fb4092e77435a750f2ad07 in 685ms, sequenceid=408, compaction requested=false 2024-12-04T06:42:01,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:01,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:42:01,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:01,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:01,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:01,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:01,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:01,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:01,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a06519e2dcea4d69ae251b35568e644c is 50, key is test_row_0/A:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:01,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:01,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:01,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:01,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742166_1342 (size=12301) 2024-12-04T06:42:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294581980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:02,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294582097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:02,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a06519e2dcea4d69ae251b35568e644c 2024-12-04T06:42:02,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c0534fcca83940c5a1aa49c11e8257cb is 50, key is test_row_0/B:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:02,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742167_1343 (size=12301) 2024-12-04T06:42:02,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c0534fcca83940c5a1aa49c11e8257cb 2024-12-04T06:42:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294582307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:02,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/c6133c1aecb7419db02f624a5872cb6e is 50, key is test_row_0/C:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742168_1344 (size=12301) 2024-12-04T06:42:02,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294582326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:02,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:02,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:02,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294582617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:02,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:02,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:02,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/c6133c1aecb7419db02f624a5872cb6e 2024-12-04T06:42:02,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a06519e2dcea4d69ae251b35568e644c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c 2024-12-04T06:42:02,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T06:42:02,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/c0534fcca83940c5a1aa49c11e8257cb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb 2024-12-04T06:42:02,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T06:42:02,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/c6133c1aecb7419db02f624a5872cb6e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e 2024-12-04T06:42:02,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T06:42:02,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ac3d355fe6fb4092e77435a750f2ad07 in 906ms, sequenceid=423, compaction requested=true 2024-12-04T06:42:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:02,737 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:02,737 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:02,738 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:02,738 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:42:02,738 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,738 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/14a605e796604a509d2b6362c430e83b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.9 K 2024-12-04T06:42:02,738 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:02,738 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:42:02,738 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,738 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6cb0b2fcdd984149a761e562bbee0ca4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=41.6 K 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a605e796604a509d2b6362c430e83b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb0b2fcdd984149a761e562bbee0ca4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 31654f34d02c46ca9e123ba952e57de3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733294521111 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80a9c2b2235a4467acb130cc1d931328, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733294521075 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c0534fcca83940c5a1aa49c11e8257cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:02,739 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06519e2dcea4d69ae251b35568e644c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:02,747 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:02,747 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/b09050b26a7c4c4c8c7b6d12ac57b395 is 50, key is test_row_0/B:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:02,750 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:02,750 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/3c56317c84934dd38cb0121c963ccee2 is 50, key is test_row_0/A:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:02,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742169_1345 (size=13255) 2024-12-04T06:42:02,771 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/b09050b26a7c4c4c8c7b6d12ac57b395 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/b09050b26a7c4c4c8c7b6d12ac57b395 2024-12-04T06:42:02,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742170_1346 (size=13255) 2024-12-04T06:42:02,776 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into b09050b26a7c4c4c8c7b6d12ac57b395(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:02,776 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:02,776 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294522737; duration=0sec 2024-12-04T06:42:02,776 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:02,776 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:42:02,776 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:02,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:02,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:42:02,777 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,777 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d26479367b66402abadeb4a0e7c95edc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=36.9 K 2024-12-04T06:42:02,777 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d26479367b66402abadeb4a0e7c95edc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733294520555 2024-12-04T06:42:02,778 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 61c2fb0544a4415491c111cdf358714e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1733294521111 2024-12-04T06:42:02,778 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c6133c1aecb7419db02f624a5872cb6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:02,784 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:02,784 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/cbc0b418374e4f18ab3cb28b2d174119 is 50, key is test_row_0/C:col10/1733294521182/Put/seqid=0 2024-12-04T06:42:02,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742171_1347 (size=13255) 2024-12-04T06:42:02,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:02,794 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:02,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/619cf6fbd3364ca5a6434948882c6137 is 50, key is test_row_0/A:col10/1733294521924/Put/seqid=0 2024-12-04T06:42:02,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742172_1348 (size=12301) 2024-12-04T06:42:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:03,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. as already flushing 2024-12-04T06:42:03,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294583170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:03,179 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/3c56317c84934dd38cb0121c963ccee2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3c56317c84934dd38cb0121c963ccee2 2024-12-04T06:42:03,184 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into 3c56317c84934dd38cb0121c963ccee2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:03,184 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:03,184 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294522737; duration=0sec 2024-12-04T06:42:03,184 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:03,184 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:42:03,198 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/cbc0b418374e4f18ab3cb28b2d174119 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cbc0b418374e4f18ab3cb28b2d174119 2024-12-04T06:42:03,203 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into cbc0b418374e4f18ab3cb28b2d174119(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:03,203 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:03,203 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294522737; duration=0sec 2024-12-04T06:42:03,203 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:03,204 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:42:03,212 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/619cf6fbd3364ca5a6434948882c6137 2024-12-04T06:42:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e3aad69a9d4e4fd99ef5430bd56c06b8 is 50, key is test_row_0/B:col10/1733294521924/Put/seqid=0 2024-12-04T06:42:03,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742173_1349 (size=12301) 2024-12-04T06:42:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294583276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42286 deadline: 1733294583337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:03,340 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in ac3d355fe6fb4092e77435a750f2ad07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:03,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294583486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:03,634 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e3aad69a9d4e4fd99ef5430bd56c06b8 2024-12-04T06:42:03,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d60e82557953465190cfe1951a27bfcb is 50, key is test_row_0/C:col10/1733294521924/Put/seqid=0 2024-12-04T06:42:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742174_1350 (size=12301) 2024-12-04T06:42:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1733294583796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:04,048 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d60e82557953465190cfe1951a27bfcb 2024-12-04T06:42:04,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/619cf6fbd3364ca5a6434948882c6137 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137 2024-12-04T06:42:04,056 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137, entries=150, sequenceid=444, filesize=12.0 K 2024-12-04T06:42:04,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e3aad69a9d4e4fd99ef5430bd56c06b8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8 2024-12-04T06:42:04,060 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8, entries=150, sequenceid=444, filesize=12.0 K 2024-12-04T06:42:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/d60e82557953465190cfe1951a27bfcb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb 2024-12-04T06:42:04,066 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb, entries=150, sequenceid=444, filesize=12.0 K 2024-12-04T06:42:04,066 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ac3d355fe6fb4092e77435a750f2ad07 in 1272ms, sequenceid=444, compaction requested=false 2024-12-04T06:42:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:04,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-04T06:42:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-04T06:42:04,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-04T06:42:04,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-12-04T06:42:04,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3520 sec 2024-12-04T06:42:04,219 INFO [Thread-1215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1215. 2024-12-04T06:42:04,219 DEBUG [Thread-1215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,219 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,219 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,220 INFO [Thread-1219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1219. 2024-12-04T06:42:04,220 DEBUG [Thread-1219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,220 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,220 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,221 INFO [Thread-1211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1211. 2024-12-04T06:42:04,221 DEBUG [Thread-1211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,221 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,221 INFO [Thread-1217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1217. 2024-12-04T06:42:04,221 DEBUG [Thread-1217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,221 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,221 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,221 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,222 INFO [Thread-1213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1213. 2024-12-04T06:42:04,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,222 DEBUG [Thread-1213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,222 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,222 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:04,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:42:04,309 INFO [Thread-1206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1206. 2024-12-04T06:42:04,309 DEBUG [Thread-1206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:04,309 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:04,309 DEBUG [Thread-1206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:04,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:04,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/cf0259033e7a40e8a1d73e578a3ed076 is 50, key is test_row_0/A:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742175_1351 (size=12301) 2024-12-04T06:42:04,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/cf0259033e7a40e8a1d73e578a3ed076 2024-12-04T06:42:04,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e7237568328441de917155f32c58a676 is 50, key is test_row_0/B:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742176_1352 (size=12301) 2024-12-04T06:42:05,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e7237568328441de917155f32c58a676 2024-12-04T06:42:05,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/42776ff82c694e59b8ec459f15177721 is 50, key is test_row_0/C:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:05,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742177_1353 (size=12301) 2024-12-04T06:42:05,358 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-12-04T06:42:05,358 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:05,358 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:05,358 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:05,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:05,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/42776ff82c694e59b8ec459f15177721 2024-12-04T06:42:05,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/cf0259033e7a40e8a1d73e578a3ed076 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076 2024-12-04T06:42:05,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076, entries=150, sequenceid=459, filesize=12.0 K 2024-12-04T06:42:05,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/e7237568328441de917155f32c58a676 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676 2024-12-04T06:42:05,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676, entries=150, sequenceid=459, filesize=12.0 K 2024-12-04T06:42:05,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/42776ff82c694e59b8ec459f15177721 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721 2024-12-04T06:42:05,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721, entries=150, sequenceid=459, filesize=12.0 K 2024-12-04T06:42:05,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for ac3d355fe6fb4092e77435a750f2ad07 in 1240ms, sequenceid=459, compaction requested=true 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:05,548 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac3d355fe6fb4092e77435a750f2ad07:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:05,548 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:05,549 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:05,549 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:05,549 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/A is initiating minor compaction (all files) 2024-12-04T06:42:05,549 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/B is initiating minor compaction (all files) 2024-12-04T06:42:05,549 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/A in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:05,549 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/B in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:05,549 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3c56317c84934dd38cb0121c963ccee2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=37.0 K 2024-12-04T06:42:05,549 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/b09050b26a7c4c4c8c7b6d12ac57b395, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=37.0 K 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b09050b26a7c4c4c8c7b6d12ac57b395, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c56317c84934dd38cb0121c963ccee2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 619cf6fbd3364ca5a6434948882c6137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733294521924 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e3aad69a9d4e4fd99ef5430bd56c06b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733294521924 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0259033e7a40e8a1d73e578a3ed076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294523137 2024-12-04T06:42:05,550 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting e7237568328441de917155f32c58a676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294523137 2024-12-04T06:42:05,556 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:05,556 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#A#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:05,556 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/be1ff863fa2b46e2aa8255505b5e8ec2 is 50, key is test_row_0/A:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:05,556 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/440dabd6204f4d1eaa7d0d7437082716 is 50, key is test_row_0/B:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:05,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742178_1354 (size=13357) 2024-12-04T06:42:05,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742179_1355 (size=13357) 2024-12-04T06:42:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-04T06:42:05,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:05,964 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/440dabd6204f4d1eaa7d0d7437082716 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/440dabd6204f4d1eaa7d0d7437082716 2024-12-04T06:42:05,964 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/be1ff863fa2b46e2aa8255505b5e8ec2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/be1ff863fa2b46e2aa8255505b5e8ec2 2024-12-04T06:42:05,968 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/B of ac3d355fe6fb4092e77435a750f2ad07 into 440dabd6204f4d1eaa7d0d7437082716(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:05,968 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/B, priority=13, startTime=1733294525548; duration=0sec 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:B 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:05,968 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/A of ac3d355fe6fb4092e77435a750f2ad07 into be1ff863fa2b46e2aa8255505b5e8ec2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:05,968 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/A, priority=13, startTime=1733294525548; duration=0sec 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:05,968 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:A 2024-12-04T06:42:05,969 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:05,969 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): ac3d355fe6fb4092e77435a750f2ad07/C is initiating minor compaction (all files) 2024-12-04T06:42:05,969 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac3d355fe6fb4092e77435a750f2ad07/C in TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:05,969 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cbc0b418374e4f18ab3cb28b2d174119, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp, totalSize=37.0 K 2024-12-04T06:42:05,969 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc0b418374e4f18ab3cb28b2d174119, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733294521177 2024-12-04T06:42:05,970 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d60e82557953465190cfe1951a27bfcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733294521924 2024-12-04T06:42:05,970 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 42776ff82c694e59b8ec459f15177721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1733294523137 2024-12-04T06:42:05,977 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac3d355fe6fb4092e77435a750f2ad07#C#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:05,978 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/3e24732805404de594c113d0bd95ebef is 50, key is test_row_0/C:col10/1733294524307/Put/seqid=0 2024-12-04T06:42:05,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742180_1356 (size=13357) 2024-12-04T06:42:06,385 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/3e24732805404de594c113d0bd95ebef as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3e24732805404de594c113d0bd95ebef 2024-12-04T06:42:06,390 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac3d355fe6fb4092e77435a750f2ad07/C of ac3d355fe6fb4092e77435a750f2ad07 into 3e24732805404de594c113d0bd95ebef(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:06,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac3d355fe6fb4092e77435a750f2ad07: 2024-12-04T06:42:06,390 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., storeName=ac3d355fe6fb4092e77435a750f2ad07/C, priority=13, startTime=1733294525548; duration=0sec 2024-12-04T06:42:06,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:06,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac3d355fe6fb4092e77435a750f2ad07:C 2024-12-04T06:42:06,908 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-12-04T06:42:06,908 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:06,908 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,909 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:06,948 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-12-04T06:42:06,948 INFO [Thread-1208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1208. 2024-12-04T06:42:06,948 DEBUG [Thread-1208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:06,948 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:06,948 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,948 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,948 DEBUG [Thread-1208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,948 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:06,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:06,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:42:06,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-04T06:42:06,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-04T06:42:06,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 161 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3035 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3160 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2937 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3035 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3152 2024-12-04T06:42:06,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:42:06,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:42:06,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:42:06,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:06,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:06,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:42:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:06,952 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294526952"}]},"ts":"1733294526952"} 2024-12-04T06:42:06,953 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:42:06,953 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:42:06,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:42:06,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, UNASSIGN}] 2024-12-04T06:42:06,956 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, UNASSIGN 2024-12-04T06:42:06,956 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ac3d355fe6fb4092e77435a750f2ad07, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:06,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, UNASSIGN because future has completed 2024-12-04T06:42:06,957 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:42:06,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:07,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:07,109 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:07,109 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:42:07,109 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing ac3d355fe6fb4092e77435a750f2ad07, disabling compactions & flushes 2024-12-04T06:42:07,109 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:07,109 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. after waiting 0 ms 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:07,110 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing ac3d355fe6fb4092e77435a750f2ad07 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=A 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=B 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac3d355fe6fb4092e77435a750f2ad07, store=C 2024-12-04T06:42:07,110 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:07,113 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a9c3344bc6cf48c3ba56313fc1974128 is 50, key is test_row_1/A:col10/1733294526947/Put/seqid=0 2024-12-04T06:42:07,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742181_1357 (size=9857) 2024-12-04T06:42:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:07,517 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a9c3344bc6cf48c3ba56313fc1974128 2024-12-04T06:42:07,523 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe16e416441f4ea4abc1862bfbf3dbfa is 50, key is test_row_1/B:col10/1733294526947/Put/seqid=0 2024-12-04T06:42:07,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742182_1358 (size=9857) 2024-12-04T06:42:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:07,926 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe16e416441f4ea4abc1862bfbf3dbfa 2024-12-04T06:42:07,932 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/230fad96a8a74384b6962d4d94b90c38 is 50, key is test_row_1/C:col10/1733294526947/Put/seqid=0 2024-12-04T06:42:07,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742183_1359 (size=9857) 2024-12-04T06:42:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:08,336 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/230fad96a8a74384b6962d4d94b90c38 2024-12-04T06:42:08,340 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/A/a9c3344bc6cf48c3ba56313fc1974128 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a9c3344bc6cf48c3ba56313fc1974128 2024-12-04T06:42:08,343 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a9c3344bc6cf48c3ba56313fc1974128, entries=100, sequenceid=469, filesize=9.6 K 2024-12-04T06:42:08,343 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/B/fe16e416441f4ea4abc1862bfbf3dbfa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe16e416441f4ea4abc1862bfbf3dbfa 2024-12-04T06:42:08,346 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe16e416441f4ea4abc1862bfbf3dbfa, entries=100, sequenceid=469, filesize=9.6 K 2024-12-04T06:42:08,347 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/.tmp/C/230fad96a8a74384b6962d4d94b90c38 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/230fad96a8a74384b6962d4d94b90c38 2024-12-04T06:42:08,349 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/230fad96a8a74384b6962d4d94b90c38, entries=100, sequenceid=469, filesize=9.6 K 2024-12-04T06:42:08,350 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ac3d355fe6fb4092e77435a750f2ad07 in 1240ms, sequenceid=469, compaction requested=false 2024-12-04T06:42:08,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b16b52db237a482cbe9e9d18d0f8e491, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c4770ba59d15427db782fb6154fece7e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3927e1f067944056ad028ae90d02619f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/2bd39c8c6c4f44af898d5ac3ba6a07dc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/72592a9c687f4df3b0bbe867628b6409, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d85da137ad3b4995912df90070aefd9c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/655e102e3f674886816996969ff48b57, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6cb0b2fcdd984149a761e562bbee0ca4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3c56317c84934dd38cb0121c963ccee2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076] to archive 2024-12-04T06:42:08,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:08,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/4f3a5329994744f6babc7e4223f3356d 2024-12-04T06:42:08,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ffab3ef8d2474910873e5aa54f0fcec3 2024-12-04T06:42:08,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6983b353a3704d47af0ace147576c255 2024-12-04T06:42:08,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b16b52db237a482cbe9e9d18d0f8e491 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b16b52db237a482cbe9e9d18d0f8e491 2024-12-04T06:42:08,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/b59bb0c228fe4eb380ac4dece1d55bc1 2024-12-04T06:42:08,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/9c58e10db03845c4b52ee49bfb624c76 2024-12-04T06:42:08,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c4770ba59d15427db782fb6154fece7e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c4770ba59d15427db782fb6154fece7e 2024-12-04T06:42:08,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/fd57f6354d364b0884f64358adce1585 2024-12-04T06:42:08,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/ba46730555814ccdac2b2327c42fdcc2 2024-12-04T06:42:08,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3927e1f067944056ad028ae90d02619f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3927e1f067944056ad028ae90d02619f 2024-12-04T06:42:08,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5dc91cb8804446678323853888010dc9 2024-12-04T06:42:08,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d87170c874e545f784849fdde6779e92 2024-12-04T06:42:08,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/5b29f289965349939b149a842e208aaa 2024-12-04T06:42:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/2bd39c8c6c4f44af898d5ac3ba6a07dc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/2bd39c8c6c4f44af898d5ac3ba6a07dc 2024-12-04T06:42:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/40866ef697d245c1986790eecfd0c3fa 2024-12-04T06:42:08,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/72592a9c687f4df3b0bbe867628b6409 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/72592a9c687f4df3b0bbe867628b6409 2024-12-04T06:42:08,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/11e569d9f7284b24af664693e2e385ce 2024-12-04T06:42:08,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/c335a3f8e43a4d49b9626c70b589d979 2024-12-04T06:42:08,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d85da137ad3b4995912df90070aefd9c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d85da137ad3b4995912df90070aefd9c 2024-12-04T06:42:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/66d41650c3624ff58979c7d65767e4a4 2024-12-04T06:42:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6df19fb1f40d42ffa2a9684ab4145ed0 2024-12-04T06:42:08,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/655e102e3f674886816996969ff48b57 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/655e102e3f674886816996969ff48b57 2024-12-04T06:42:08,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/0a78568323a1433d979c9f6748b12e2b 2024-12-04T06:42:08,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/d3141cc2c2374fd59b563627eec9e8bf 2024-12-04T06:42:08,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/db22e06ed32449ebba2803017c26168c 2024-12-04T06:42:08,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6cb0b2fcdd984149a761e562bbee0ca4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/6cb0b2fcdd984149a761e562bbee0ca4 2024-12-04T06:42:08,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/80a9c2b2235a4467acb130cc1d931328 2024-12-04T06:42:08,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3c56317c84934dd38cb0121c963ccee2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/3c56317c84934dd38cb0121c963ccee2 2024-12-04T06:42:08,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a06519e2dcea4d69ae251b35568e644c 2024-12-04T06:42:08,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/619cf6fbd3364ca5a6434948882c6137 2024-12-04T06:42:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/cf0259033e7a40e8a1d73e578a3ed076 2024-12-04T06:42:08,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/1b4e93570f7e4c7092301920f2c61afc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bba15cc23002482f8fb602209ad56e07, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/d8e2977bb35f4dc693c8b0e60ec64119, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/8798d64ff1884a489b3e83af566b0898, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/941fa3d28bba4defbb64a57a180469a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/dc3bc1d9fd184a6ea39fd88c9d01f558, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/5d1ec6cd12844eca81716bae6617e5c7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/14a605e796604a509d2b6362c430e83b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/b09050b26a7c4c4c8c7b6d12ac57b395, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676] to archive 2024-12-04T06:42:08,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:08,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9b34bb0edf0a4007ae041e9a1eba13c3 2024-12-04T06:42:08,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe2efc1dcbfe45f49ca673dbab8dd54d 2024-12-04T06:42:08,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/1b4e93570f7e4c7092301920f2c61afc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/1b4e93570f7e4c7092301920f2c61afc 2024-12-04T06:42:08,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/40ff60bbd30045b3911ef56f2bb00510 2024-12-04T06:42:08,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/cd3b93d6a24c4b31b09cb878a98dd02b 2024-12-04T06:42:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bba15cc23002482f8fb602209ad56e07 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bba15cc23002482f8fb602209ad56e07 2024-12-04T06:42:08,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6dd246dc6b64427490b5dbf019734c94 2024-12-04T06:42:08,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85955a16bb1e4c9f914dcfb4cb5d4554 2024-12-04T06:42:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/092ab851e0ad41d9988efb40fcdda309 2024-12-04T06:42:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/d8e2977bb35f4dc693c8b0e60ec64119 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/d8e2977bb35f4dc693c8b0e60ec64119 2024-12-04T06:42:08,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/85927eb7585e4c6f8834a2370012af2b 2024-12-04T06:42:08,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/7b57fe3f29b443a296bf86e277716dae 2024-12-04T06:42:08,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/8798d64ff1884a489b3e83af566b0898 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/8798d64ff1884a489b3e83af566b0898 2024-12-04T06:42:08,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/bcac7db8776347898599cabaf30a8b4c 2024-12-04T06:42:08,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c28f269691a74757a94b7cfce4608cad 2024-12-04T06:42:08,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/941fa3d28bba4defbb64a57a180469a3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/941fa3d28bba4defbb64a57a180469a3 2024-12-04T06:42:08,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/9e6061765a194c9dbddb231e194e3d5b 2024-12-04T06:42:08,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/26e42ccf91914d8a98f035f446403023 2024-12-04T06:42:08,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/dc3bc1d9fd184a6ea39fd88c9d01f558 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/dc3bc1d9fd184a6ea39fd88c9d01f558 2024-12-04T06:42:08,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6fac4d5663b34be99a45ef76bd8ae1e9 2024-12-04T06:42:08,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/3f9114ab46a54fde8013facd0c08ed7f 2024-12-04T06:42:08,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/5d1ec6cd12844eca81716bae6617e5c7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/5d1ec6cd12844eca81716bae6617e5c7 2024-12-04T06:42:08,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c9272659d6514f46adce8f53b6165f10 2024-12-04T06:42:08,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/007fae864b6a4f1bb0b425a37ec8de15 2024-12-04T06:42:08,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/14a605e796604a509d2b6362c430e83b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/14a605e796604a509d2b6362c430e83b 2024-12-04T06:42:08,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/6470dbd7d30341828b4a7da0f976337d 2024-12-04T06:42:08,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/31654f34d02c46ca9e123ba952e57de3 2024-12-04T06:42:08,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/b09050b26a7c4c4c8c7b6d12ac57b395 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/b09050b26a7c4c4c8c7b6d12ac57b395 2024-12-04T06:42:08,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/c0534fcca83940c5a1aa49c11e8257cb 2024-12-04T06:42:08,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e3aad69a9d4e4fd99ef5430bd56c06b8 2024-12-04T06:42:08,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/e7237568328441de917155f32c58a676 2024-12-04T06:42:08,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/09a5184059fb4f15b48c7b6d3cf4c8e2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cb5a4516ffda4885bfde916bde2140f5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e22c912d985a4e95884dacd5fab31790, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3c81816a2b0a4dc3919a1b74db665acc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c97123072ae149dba8bb647b94a31aa1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/2ce8da0b537f45f49503534311994de9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4df28b0e62e4ff784243b697005795f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d26479367b66402abadeb4a0e7c95edc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cbc0b418374e4f18ab3cb28b2d174119, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721] to archive 2024-12-04T06:42:08,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:08,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d385a8df07a945acb87b7b39eaa23435 2024-12-04T06:42:08,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6237c13ad66b40248df54b2f4d502e94 2024-12-04T06:42:08,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/09a5184059fb4f15b48c7b6d3cf4c8e2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/09a5184059fb4f15b48c7b6d3cf4c8e2 2024-12-04T06:42:08,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e835f50a1b7d486e94f438da1bcd6d47 2024-12-04T06:42:08,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/851cf2e0b6384871a8cfde8fc1b44884 2024-12-04T06:42:08,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cb5a4516ffda4885bfde916bde2140f5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cb5a4516ffda4885bfde916bde2140f5 2024-12-04T06:42:08,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ebb14b9dbf8a411480a6f8491e413db5 2024-12-04T06:42:08,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/bb9114e4ee0641f4bfe307f25b2bf449 2024-12-04T06:42:08,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a5e69064120645748ba0e9c4d80cf228 2024-12-04T06:42:08,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e22c912d985a4e95884dacd5fab31790 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/e22c912d985a4e95884dacd5fab31790 2024-12-04T06:42:08,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/8fdae2ee689e4aac952c673f78e2d686 2024-12-04T06:42:08,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4d9ad19e259478c866feae0bfe0fc06 2024-12-04T06:42:08,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3c81816a2b0a4dc3919a1b74db665acc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3c81816a2b0a4dc3919a1b74db665acc 2024-12-04T06:42:08,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/ffc556ac55fa43a0a2a2227ede0ba971 2024-12-04T06:42:08,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/33bab05fc4f14038835eb945f44d87b3 2024-12-04T06:42:08,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c97123072ae149dba8bb647b94a31aa1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c97123072ae149dba8bb647b94a31aa1 2024-12-04T06:42:08,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/6ac655ff96bf44758e9e3869c75436f8 2024-12-04T06:42:08,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/5287d8bdcbf84430bd78bd05b222a1e4 2024-12-04T06:42:08,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/2ce8da0b537f45f49503534311994de9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/2ce8da0b537f45f49503534311994de9 2024-12-04T06:42:08,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/676dfbad9b7947daa6bb63c2ea9ccebe 2024-12-04T06:42:08,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/b08fbca940044b6da5d36511dfeea5b6 2024-12-04T06:42:08,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4df28b0e62e4ff784243b697005795f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/a4df28b0e62e4ff784243b697005795f 2024-12-04T06:42:08,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f59735b559ee4aa7a232dbed2871e35c 2024-12-04T06:42:08,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d45e77153e8d467098627cbeb418f854 2024-12-04T06:42:08,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d26479367b66402abadeb4a0e7c95edc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d26479367b66402abadeb4a0e7c95edc 2024-12-04T06:42:08,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/f46a299217414c039fd36a6b5f564673 2024-12-04T06:42:08,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/61c2fb0544a4415491c111cdf358714e 2024-12-04T06:42:08,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cbc0b418374e4f18ab3cb28b2d174119 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/cbc0b418374e4f18ab3cb28b2d174119 2024-12-04T06:42:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/c6133c1aecb7419db02f624a5872cb6e 2024-12-04T06:42:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/d60e82557953465190cfe1951a27bfcb 2024-12-04T06:42:08,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/42776ff82c694e59b8ec459f15177721 2024-12-04T06:42:08,437 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-12-04T06:42:08,437 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. 2024-12-04T06:42:08,437 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for ac3d355fe6fb4092e77435a750f2ad07: Waiting for close lock at 1733294527109Running coprocessor pre-close hooks at 1733294527109Disabling compacts and flushes for region at 1733294527109Disabling writes for close at 1733294527110 (+1 ms)Obtaining lock to block concurrent updates at 1733294527110Preparing flush snapshotting stores in ac3d355fe6fb4092e77435a750f2ad07 at 1733294527110Finished memstore snapshotting TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733294527110Flushing stores of TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07. at 1733294527110Flushing ac3d355fe6fb4092e77435a750f2ad07/A: creating writer at 1733294527111 (+1 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/A: appending metadata at 1733294527113 (+2 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/A: closing flushed file at 1733294527113Flushing ac3d355fe6fb4092e77435a750f2ad07/B: creating writer at 1733294527520 (+407 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/B: appending metadata at 1733294527522 (+2 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/B: closing flushed file at 1733294527522Flushing ac3d355fe6fb4092e77435a750f2ad07/C: creating writer at 1733294527930 (+408 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/C: appending metadata at 1733294527932 (+2 ms)Flushing ac3d355fe6fb4092e77435a750f2ad07/C: closing flushed file at 1733294527932Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ce6b2cd: reopening flushed file at 1733294528339 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15848e9d: reopening flushed file at 1733294528343 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e31444: reopening flushed file at 1733294528346 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ac3d355fe6fb4092e77435a750f2ad07 in 1240ms, sequenceid=469, compaction requested=false at 1733294528350 (+4 ms)Writing region close event to WAL at 1733294528434 (+84 ms)Running coprocessor post-close hooks at 1733294528437 (+3 ms)Closed at 1733294528437 2024-12-04T06:42:08,439 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:08,439 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ac3d355fe6fb4092e77435a750f2ad07, regionState=CLOSED 2024-12-04T06:42:08,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:08,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-04T06:42:08,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure ac3d355fe6fb4092e77435a750f2ad07, server=607fd5c6574c,38257,1733294430862 in 1.4840 sec 2024-12-04T06:42:08,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T06:42:08,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac3d355fe6fb4092e77435a750f2ad07, UNASSIGN in 1.4880 sec 2024-12-04T06:42:08,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-04T06:42:08,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4910 sec 2024-12-04T06:42:08,447 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294528447"}]},"ts":"1733294528447"} 2024-12-04T06:42:08,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:42:08,448 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:42:08,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4990 sec 2024-12-04T06:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-04T06:42:09,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:09,108 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:42:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,109 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T06:42:09,110 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,111 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:09,113 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/recovered.edits] 2024-12-04T06:42:09,115 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a9c3344bc6cf48c3ba56313fc1974128 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/a9c3344bc6cf48c3ba56313fc1974128 2024-12-04T06:42:09,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/be1ff863fa2b46e2aa8255505b5e8ec2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/A/be1ff863fa2b46e2aa8255505b5e8ec2 2024-12-04T06:42:09,117 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/440dabd6204f4d1eaa7d0d7437082716 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/440dabd6204f4d1eaa7d0d7437082716 2024-12-04T06:42:09,118 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe16e416441f4ea4abc1862bfbf3dbfa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/B/fe16e416441f4ea4abc1862bfbf3dbfa 2024-12-04T06:42:09,120 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/230fad96a8a74384b6962d4d94b90c38 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/230fad96a8a74384b6962d4d94b90c38 2024-12-04T06:42:09,120 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3e24732805404de594c113d0bd95ebef to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/C/3e24732805404de594c113d0bd95ebef 2024-12-04T06:42:09,122 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/recovered.edits/472.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07/recovered.edits/472.seqid 2024-12-04T06:42:09,123 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/ac3d355fe6fb4092e77435a750f2ad07 2024-12-04T06:42:09,123 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:42:09,124 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,126 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:42:09,128 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:42:09,129 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,129 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:42:09,129 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294529129"}]},"ts":"9223372036854775807"} 2024-12-04T06:42:09,131 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:42:09,131 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ac3d355fe6fb4092e77435a750f2ad07, NAME => 'TestAcidGuarantees,,1733294501972.ac3d355fe6fb4092e77435a750f2ad07.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:42:09,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:42:09,131 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294529131"}]},"ts":"9223372036854775807"} 2024-12-04T06:42:09,132 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:42:09,133 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-12-04T06:42:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-04T06:42:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:42:09,217 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:09,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 249), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 419), ProcessCount=11 (was 11), AvailableMemoryMB=5993 (was 6544) 2024-12-04T06:42:09,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=5993 2024-12-04T06:42:09,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:42:09,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:42:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:09,237 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:42:09,238 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:09,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-04T06:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:09,238 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:42:09,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742184_1360 (size=1061) 2024-12-04T06:42:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:09,645 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:42:09,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742185_1361 (size=53) 2024-12-04T06:42:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 896711ea717d7f14b2e304328616f183, disabling compactions & flushes 2024-12-04T06:42:10,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. after waiting 0 ms 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,050 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 896711ea717d7f14b2e304328616f183: Waiting for close lock at 1733294530050Disabling compacts and flushes for region at 1733294530050Disabling writes for close at 1733294530050Writing region close event to WAL at 1733294530050Closed at 1733294530050 2024-12-04T06:42:10,051 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:42:10,051 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294530051"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294530051"}]},"ts":"1733294530051"} 2024-12-04T06:42:10,053 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:42:10,053 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:42:10,053 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294530053"}]},"ts":"1733294530053"} 2024-12-04T06:42:10,054 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:42:10,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, ASSIGN}] 2024-12-04T06:42:10,055 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, ASSIGN 2024-12-04T06:42:10,056 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:42:10,207 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:10,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, ASSIGN because future has completed 2024-12-04T06:42:10,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:10,363 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,363 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:42:10,364 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,364 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:42:10,364 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,364 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,365 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,366 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:10,366 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName A 2024-12-04T06:42:10,366 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:10,366 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:10,367 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,367 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:10,367 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName B 2024-12-04T06:42:10,367 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:10,368 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:10,368 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,369 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:10,369 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName C 2024-12-04T06:42:10,369 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:10,369 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:10,369 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,369 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,370 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,370 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,371 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,371 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,371 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:42:10,372 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,374 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:42:10,374 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 896711ea717d7f14b2e304328616f183; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60330820, jitterRate=-0.10100072622299194}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:42:10,374 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:10,374 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 896711ea717d7f14b2e304328616f183: Running coprocessor pre-open hook at 1733294530364Writing region info on filesystem at 1733294530364Initializing all the Stores at 1733294530364Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294530364Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294530365 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294530365Cleaning up temporary data from old regions at 1733294530371 (+6 ms)Running coprocessor post-open hooks at 1733294530374 (+3 ms)Region opened successfully at 1733294530374 2024-12-04T06:42:10,375 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., pid=87, masterSystemTime=1733294530360 2024-12-04T06:42:10,377 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,377 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:10,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:10,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:10,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-04T06:42:10,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 in 170 msec 2024-12-04T06:42:10,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-04T06:42:10,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, ASSIGN in 326 msec 2024-12-04T06:42:10,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:42:10,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294530383"}]},"ts":"1733294530383"} 2024-12-04T06:42:10,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:42:10,385 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:42:10,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-04T06:42:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T06:42:11,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:11,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370e5137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:11,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:11,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:11,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a76c47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:11,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11430e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:11,401 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:11,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:11,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:11,403 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:42:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:42:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@96387ed 2024-12-04T06:42:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:42:11,405 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:42:11,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:42:11,406 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:42:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:11,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742186_1362 (size=1097) 2024-12-04T06:42:11,818 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T06:42:11,818 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T06:42:11,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:42:11,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE}] 2024-12-04T06:42:11,824 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE 2024-12-04T06:42:11,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:11,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE because future has completed 2024-12-04T06:42:11,827 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:42:11,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:11,979 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:11,979 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:42:11,979 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 896711ea717d7f14b2e304328616f183, disabling compactions & flushes 2024-12-04T06:42:11,979 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:11,979 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:11,979 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. after waiting 0 ms 2024-12-04T06:42:11,979 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:11,983 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T06:42:11,984 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:11,984 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 896711ea717d7f14b2e304328616f183: Waiting for close lock at 1733294531979Running coprocessor pre-close hooks at 1733294531979Disabling compacts and flushes for region at 1733294531979Disabling writes for close at 1733294531979Writing region close event to WAL at 1733294531980 (+1 ms)Running coprocessor post-close hooks at 1733294531984 (+4 ms)Closed at 1733294531984 2024-12-04T06:42:11,984 WARN [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 896711ea717d7f14b2e304328616f183 to self. 2024-12-04T06:42:11,987 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:11,987 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=CLOSED 2024-12-04T06:42:11,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:11,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-04T06:42:11,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 in 163 msec 2024-12-04T06:42:11,993 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE; state=CLOSED, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=true 2024-12-04T06:42:12,144 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE because future has completed 2024-12-04T06:42:12,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:12,300 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,300 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:42:12,300 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,300 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:42:12,301 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,301 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,302 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,302 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:12,303 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName A 2024-12-04T06:42:12,303 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:12,304 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:12,304 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,305 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:12,305 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName B 2024-12-04T06:42:12,305 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:12,305 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:12,305 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,306 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:12,306 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 896711ea717d7f14b2e304328616f183 columnFamilyName C 2024-12-04T06:42:12,306 DEBUG [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:12,306 INFO [StoreOpener-896711ea717d7f14b2e304328616f183-1 {}] regionserver.HStore(327): Store=896711ea717d7f14b2e304328616f183/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:12,306 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,306 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,307 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,307 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,308 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,308 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,308 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:42:12,309 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,310 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 896711ea717d7f14b2e304328616f183; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59048193, jitterRate=-0.12011335790157318}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:42:12,310 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,311 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 896711ea717d7f14b2e304328616f183: Running coprocessor pre-open hook at 1733294532301Writing region info on filesystem at 1733294532301Initializing all the Stores at 1733294532301Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294532301Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294532301Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294532301Cleaning up temporary data from old regions at 1733294532308 (+7 ms)Running coprocessor post-open hooks at 1733294532310 (+2 ms)Region opened successfully at 1733294532311 (+1 ms) 2024-12-04T06:42:12,312 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., pid=92, masterSystemTime=1733294532297 2024-12-04T06:42:12,313 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,313 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,314 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=OPEN, openSeqNum=5, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:12,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-04T06:42:12,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 in 169 msec 2024-12-04T06:42:12,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T06:42:12,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, REOPEN/MOVE in 494 msec 2024-12-04T06:42:12,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-04T06:42:12,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-12-04T06:42:12,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 913 msec 2024-12-04T06:42:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-04T06:42:12,322 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:12,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d6520c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517aada4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32893b5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746e823f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24dcb63e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5036ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acb64fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,330 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2834de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1422378d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5513552f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb674e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a01bdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5745c2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62595c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c05f65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4ab058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,338 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7306938c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60170a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4fb7f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,341 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3cb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee5c99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1483dc7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,344 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e6c6b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ae4f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78bf9cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,347 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266906c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,348 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ff173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3074b693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:12,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:12,350 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bd59b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:12,351 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:12,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109822fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:12,352 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,352 DEBUG [Thread-1647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,352 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,353 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,353 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:12,353 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,354 DEBUG [Thread-1641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-04T06:42:12,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,355 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,355 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,356 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,357 DEBUG [Thread-1645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:12,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:12,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,361 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:12,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:42:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:12,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a135276b276b4d7eb30a0b6e9d2c07a5_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294532361/Put/seqid=0 2024-12-04T06:42:12,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742187_1363 (size=14594) 2024-12-04T06:42:12,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294592399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294532393, max timestamp=1733294532406Num uniques 859; 2024-12-04T06:42:12,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294532393, max timestamp=1733294532406Num uniques 859; 2024-12-04T06:42:12,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:42:12,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:42:12,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733294532393, max timestamp=1733294532406Num uniques 859; 2024-12-04T06:42:12,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-04T06:42:12,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294592438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294592439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294592440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294592442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:12,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:12,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:12,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294592516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294592557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294592737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294592777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294592777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294592777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294592777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:12,802 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:12,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a135276b276b4d7eb30a0b6e9d2c07a5_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a135276b276b4d7eb30a0b6e9d2c07a5_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:12,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/e65a6c02c82849918c3a7eb2db97dbb3, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:12,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/e65a6c02c82849918c3a7eb2db97dbb3 is 175, key is test_row_0/A:col10/1733294532361/Put/seqid=0 2024-12-04T06:42:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742188_1364 (size=39549) 2024-12-04T06:42:12,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:12,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,821 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/e65a6c02c82849918c3a7eb2db97dbb3 2024-12-04T06:42:12,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/9f351a9327164a4faefd77e54972bff7 is 50, key is test_row_0/B:col10/1733294532361/Put/seqid=0 2024-12-04T06:42:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742189_1365 (size=12001) 2024-12-04T06:42:12,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:12,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294593057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294593096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294593096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294593096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294593097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:13,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/9f351a9327164a4faefd77e54972bff7 2024-12-04T06:42:13,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:13,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:13,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3f23d48c6f384f3f9353d320446af48d is 50, key is test_row_0/C:col10/1733294532361/Put/seqid=0 2024-12-04T06:42:13,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742190_1366 (size=12001) 2024-12-04T06:42:13,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3f23d48c6f384f3f9353d320446af48d 2024-12-04T06:42:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/e65a6c02c82849918c3a7eb2db97dbb3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3 2024-12-04T06:42:13,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3, entries=200, sequenceid=15, filesize=38.6 K 2024-12-04T06:42:13,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/9f351a9327164a4faefd77e54972bff7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7 2024-12-04T06:42:13,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T06:42:13,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3f23d48c6f384f3f9353d320446af48d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d 2024-12-04T06:42:13,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-04T06:42:13,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 896711ea717d7f14b2e304328616f183 in 943ms, sequenceid=15, compaction requested=false 2024-12-04T06:42:13,310 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-04T06:42:13,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:13,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-04T06:42:13,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:13,433 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:13,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bde22056e13040c6b06b29b1ff3d84ce_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294532439/Put/seqid=0 2024-12-04T06:42:13,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742191_1367 (size=12154) 2024-12-04T06:42:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:13,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:13,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294593617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294593618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294593654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294593654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294593654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294593756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294593776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294593777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294593777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:13,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:13,852 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bde22056e13040c6b06b29b1ff3d84ce_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bde22056e13040c6b06b29b1ff3d84ce_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ee625d7480814982a2005d77a6a853d2, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:13,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ee625d7480814982a2005d77a6a853d2 is 175, key is test_row_0/A:col10/1733294532439/Put/seqid=0 2024-12-04T06:42:13,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742192_1368 (size=30955) 2024-12-04T06:42:13,858 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ee625d7480814982a2005d77a6a853d2 2024-12-04T06:42:13,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/de87aee153074abb963b699137649819 is 50, key is test_row_0/B:col10/1733294532439/Put/seqid=0 2024-12-04T06:42:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742193_1369 (size=12001) 2024-12-04T06:42:13,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:42:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294593966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294593997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294593997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294593997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,271 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/de87aee153074abb963b699137649819 2024-12-04T06:42:14,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/b83c37f676a249d7b3c2f635d1e4a673 is 50, key is test_row_0/C:col10/1733294532439/Put/seqid=0 2024-12-04T06:42:14,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742194_1370 (size=12001) 2024-12-04T06:42:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294594287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,289 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/b83c37f676a249d7b3c2f635d1e4a673 2024-12-04T06:42:14,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ee625d7480814982a2005d77a6a853d2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2 2024-12-04T06:42:14,296 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2, entries=150, sequenceid=49, filesize=30.2 K 2024-12-04T06:42:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/de87aee153074abb963b699137649819 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819 2024-12-04T06:42:14,300 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T06:42:14,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/b83c37f676a249d7b3c2f635d1e4a673 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673 2024-12-04T06:42:14,305 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673, entries=150, sequenceid=49, filesize=11.7 K 2024-12-04T06:42:14,306 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 896711ea717d7f14b2e304328616f183 in 873ms, sequenceid=49, compaction requested=false 2024-12-04T06:42:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:14,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-04T06:42:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-04T06:42:14,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T06:42:14,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9510 sec 2024-12-04T06:42:14,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.9570 sec 2024-12-04T06:42:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:14,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:14,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204653ed11087774bf7ab23f847cf4dc900_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742195_1371 (size=14594) 2024-12-04T06:42:14,341 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:14,344 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204653ed11087774bf7ab23f847cf4dc900_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204653ed11087774bf7ab23f847cf4dc900_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:14,345 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/0235a9e24e744c0d91c905f089db646c, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/0235a9e24e744c0d91c905f089db646c is 175, key is test_row_0/A:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742196_1372 (size=39549) 2024-12-04T06:42:14,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/0235a9e24e744c0d91c905f089db646c 2024-12-04T06:42:14,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/56e19e88421749b9adb106ea97afdf52 is 50, key is test_row_0/B:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742197_1373 (size=12001) 2024-12-04T06:42:14,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/56e19e88421749b9adb106ea97afdf52 2024-12-04T06:42:14,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9e8b2992093e470ebf25d3368a74f394 is 50, key is test_row_0/C:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742198_1374 (size=12001) 2024-12-04T06:42:14,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9e8b2992093e470ebf25d3368a74f394 2024-12-04T06:42:14,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/0235a9e24e744c0d91c905f089db646c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c 2024-12-04T06:42:14,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c, entries=200, sequenceid=62, filesize=38.6 K 2024-12-04T06:42:14,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/56e19e88421749b9adb106ea97afdf52 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52 2024-12-04T06:42:14,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52, entries=150, sequenceid=62, filesize=11.7 K 2024-12-04T06:42:14,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9e8b2992093e470ebf25d3368a74f394 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394 2024-12-04T06:42:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294594373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394, entries=150, sequenceid=62, filesize=11.7 K 2024-12-04T06:42:14,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 896711ea717d7f14b2e304328616f183 in 65ms, sequenceid=62, compaction requested=true 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:14,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:14,389 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:14,389 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/A is initiating minor compaction (all files) 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/B is initiating minor compaction (all files) 2024-12-04T06:42:14,390 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/A in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,390 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/B in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,390 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=107.5 K 2024-12-04T06:42:14,390 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.2 K 2024-12-04T06:42:14,390 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c] 2024-12-04T06:42:14,390 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f351a9327164a4faefd77e54972bff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294532361 2024-12-04T06:42:14,391 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65a6c02c82849918c3a7eb2db97dbb3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294532361 2024-12-04T06:42:14,391 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting de87aee153074abb963b699137649819, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733294532437 2024-12-04T06:42:14,391 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee625d7480814982a2005d77a6a853d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733294532437 2024-12-04T06:42:14,391 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e19e88421749b9adb106ea97afdf52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:14,391 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0235a9e24e744c0d91c905f089db646c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:14,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:42:14,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:14,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:14,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:14,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a1e5ff21d64542d38ca6c0f9bb9a7761_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294534390/Put/seqid=0 2024-12-04T06:42:14,403 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:14,404 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b2dbd64aca17411e850a19fcba3bfd5e is 50, key is test_row_0/B:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,405 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,406 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204f3ff9ab5fabb4ad89cd0999e3da61595_896711ea717d7f14b2e304328616f183 store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,408 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204f3ff9ab5fabb4ad89cd0999e3da61595_896711ea717d7f14b2e304328616f183, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,409 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204f3ff9ab5fabb4ad89cd0999e3da61595_896711ea717d7f14b2e304328616f183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742199_1375 (size=14594) 2024-12-04T06:42:14,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742200_1376 (size=12104) 2024-12-04T06:42:14,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:14,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742201_1377 (size=4469) 2024-12-04T06:42:14,418 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#A#compaction#325 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:14,418 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/25bfb958de85478aacbd5072154c9e1d is 175, key is test_row_0/A:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204a1e5ff21d64542d38ca6c0f9bb9a7761_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a1e5ff21d64542d38ca6c0f9bb9a7761_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:14,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7fbf46b5570d472e81a4d65ecb675d72, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742202_1378 (size=31058) 2024-12-04T06:42:14,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7fbf46b5570d472e81a4d65ecb675d72 is 175, key is test_row_0/A:col10/1733294534390/Put/seqid=0 2024-12-04T06:42:14,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742203_1379 (size=39549) 2024-12-04T06:42:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294594431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294594432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T06:42:14,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:14,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-04T06:42:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:14,509 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:14,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:14,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294594547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294594556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:14,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:14,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294594656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,662 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294594707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294594766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294594767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294594797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:14,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,819 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b2dbd64aca17411e850a19fcba3bfd5e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b2dbd64aca17411e850a19fcba3bfd5e 2024-12-04T06:42:14,824 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/B of 896711ea717d7f14b2e304328616f183 into b2dbd64aca17411e850a19fcba3bfd5e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:14,824 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:14,824 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/B, priority=13, startTime=1733294534389; duration=0sec 2024-12-04T06:42:14,824 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:14,824 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:B 2024-12-04T06:42:14,824 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:14,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:14,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/C is initiating minor compaction (all files) 2024-12-04T06:42:14,826 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/C in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,826 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.2 K 2024-12-04T06:42:14,826 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7fbf46b5570d472e81a4d65ecb675d72 2024-12-04T06:42:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:14,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f23d48c6f384f3f9353d320446af48d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294532361 2024-12-04T06:42:14,827 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b83c37f676a249d7b3c2f635d1e4a673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733294532437 2024-12-04T06:42:14,827 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8b2992093e470ebf25d3368a74f394, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:14,828 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/25bfb958de85478aacbd5072154c9e1d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d 2024-12-04T06:42:14,833 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/A of 896711ea717d7f14b2e304328616f183 into 25bfb958de85478aacbd5072154c9e1d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:14,833 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:14,833 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/A, priority=13, startTime=1733294534389; duration=0sec 2024-12-04T06:42:14,833 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:14,833 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:A 2024-12-04T06:42:14,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/1c3794ea35e44b5fb417bc59d3366f78 is 50, key is test_row_0/B:col10/1733294534390/Put/seqid=0 2024-12-04T06:42:14,842 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:14,842 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/0e2c6e67dbcb4e53a2297b62a0558a37 is 50, key is test_row_0/C:col10/1733294534323/Put/seqid=0 2024-12-04T06:42:14,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742204_1380 (size=12001) 2024-12-04T06:42:14,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742205_1381 (size=12104) 2024-12-04T06:42:14,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:14,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294595027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294595086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294595087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:15,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:15,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:15,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:15,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:15,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/1c3794ea35e44b5fb417bc59d3366f78 2024-12-04T06:42:15,258 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/0e2c6e67dbcb4e53a2297b62a0558a37 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0e2c6e67dbcb4e53a2297b62a0558a37 2024-12-04T06:42:15,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a7ec0079517b47e3b9da63068fc1f2da is 50, key is test_row_0/C:col10/1733294534390/Put/seqid=0 2024-12-04T06:42:15,271 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/C of 896711ea717d7f14b2e304328616f183 into 0e2c6e67dbcb4e53a2297b62a0558a37(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:15,272 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:15,272 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/C, priority=13, startTime=1733294534389; duration=0sec 2024-12-04T06:42:15,272 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:15,272 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:C 2024-12-04T06:42:15,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:15,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742206_1382 (size=12001) 2024-12-04T06:42:15,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a7ec0079517b47e3b9da63068fc1f2da 2024-12-04T06:42:15,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7fbf46b5570d472e81a4d65ecb675d72 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72 2024-12-04T06:42:15,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72, entries=200, sequenceid=82, filesize=38.6 K 2024-12-04T06:42:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/1c3794ea35e44b5fb417bc59d3366f78 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78 2024-12-04T06:42:15,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T06:42:15,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a7ec0079517b47e3b9da63068fc1f2da as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da 2024-12-04T06:42:15,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da, entries=150, sequenceid=82, filesize=11.7 K 2024-12-04T06:42:15,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 896711ea717d7f14b2e304328616f183 in 923ms, sequenceid=82, compaction requested=false 2024-12-04T06:42:15,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:15,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:15,429 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120434fd33389129466286ba3a5bf9055578_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294534422/Put/seqid=0 2024-12-04T06:42:15,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742207_1383 (size=12154) 2024-12-04T06:42:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:15,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:15,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294595659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294535612, max timestamp=1733294535658Num uniques 859; 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294535612, max timestamp=1733294535658Num uniques 859; 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294535612, max timestamp=1733294535658Num uniques 859; 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:15,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294595741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294595752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294595767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294595816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,829 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:15,843 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120434fd33389129466286ba3a5bf9055578_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd33389129466286ba3a5bf9055578_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:15,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/194437972e6b44ebad2c19a8e67fef1c, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:15,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/194437972e6b44ebad2c19a8e67fef1c is 175, key is test_row_0/A:col10/1733294534422/Put/seqid=0 2024-12-04T06:42:15,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742208_1384 (size=30955) 2024-12-04T06:42:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294595857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294595877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:15,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294595987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294596077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294596096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,250 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/194437972e6b44ebad2c19a8e67fef1c 2024-12-04T06:42:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/4a694a124c7f4a2faaebd062322b6c35 is 50, key is test_row_0/B:col10/1733294534422/Put/seqid=0 2024-12-04T06:42:16,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742209_1385 (size=12001) 2024-12-04T06:42:16,267 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/4a694a124c7f4a2faaebd062322b6c35 2024-12-04T06:42:16,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/d157c4e5f176486ea8c1c16578358772 is 50, key is test_row_0/C:col10/1733294534422/Put/seqid=0 2024-12-04T06:42:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742210_1386 (size=12001) 2024-12-04T06:42:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294596307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294596397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294596417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:16,689 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/d157c4e5f176486ea8c1c16578358772 2024-12-04T06:42:16,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/194437972e6b44ebad2c19a8e67fef1c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c 2024-12-04T06:42:16,697 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c, entries=150, sequenceid=97, filesize=30.2 K 2024-12-04T06:42:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/4a694a124c7f4a2faaebd062322b6c35 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35 2024-12-04T06:42:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294596687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,701 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T06:42:16,701 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/d157c4e5f176486ea8c1c16578358772 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772 2024-12-04T06:42:16,706 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772, entries=150, sequenceid=97, filesize=11.7 K 2024-12-04T06:42:16,706 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 896711ea717d7f14b2e304328616f183 in 1277ms, sequenceid=97, compaction requested=true 2024-12-04T06:42:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T06:42:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T06:42:16,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T06:42:16,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1980 sec 2024-12-04T06:42:16,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.2030 sec 2024-12-04T06:42:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:16,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:16,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120408f9e7de8ea0436caff431940ba7c7a6_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742211_1387 (size=14794) 2024-12-04T06:42:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294596917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294596928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:16,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294596936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294597036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294597036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:17,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120408f9e7de8ea0436caff431940ba7c7a6_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120408f9e7de8ea0436caff431940ba7c7a6_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:17,250 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/a48aff79e7e44b4198fce3b5e72aafc8, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:17,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/a48aff79e7e44b4198fce3b5e72aafc8 is 175, key is test_row_0/A:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:17,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742212_1388 (size=39749) 2024-12-04T06:42:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294597256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294597256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294597576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294597577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,656 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/a48aff79e7e44b4198fce3b5e72aafc8 2024-12-04T06:42:17,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ab8fa247474047bfa7216993b13fa69f is 50, key is test_row_0/B:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:17,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742213_1389 (size=12151) 2024-12-04T06:42:17,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294597836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,846 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:17,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294597956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:17,964 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:18,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ab8fa247474047bfa7216993b13fa69f 2024-12-04T06:42:18,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/92c791cd6ece47ea8cde526874d33cda is 50, key is test_row_0/C:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:18,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742214_1390 (size=12151) 2024-12-04T06:42:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294598097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294598097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:18,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/92c791cd6ece47ea8cde526874d33cda 2024-12-04T06:42:18,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/a48aff79e7e44b4198fce3b5e72aafc8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8 2024-12-04T06:42:18,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8, entries=200, sequenceid=133, filesize=38.8 K 2024-12-04T06:42:18,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ab8fa247474047bfa7216993b13fa69f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f 2024-12-04T06:42:18,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f, entries=150, sequenceid=133, filesize=11.9 K 2024-12-04T06:42:18,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/92c791cd6ece47ea8cde526874d33cda as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda 2024-12-04T06:42:18,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda, entries=150, sequenceid=133, filesize=11.9 K 2024-12-04T06:42:18,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 896711ea717d7f14b2e304328616f183 in 1662ms, sequenceid=133, compaction requested=true 2024-12-04T06:42:18,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:18,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:18,496 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:18,496 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/B is initiating minor compaction (all files) 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/A is initiating minor compaction (all files) 2024-12-04T06:42:18,497 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/B in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:18,497 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/A in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:18,497 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b2dbd64aca17411e850a19fcba3bfd5e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=47.1 K 2024-12-04T06:42:18,497 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=138.0 K 2024-12-04T06:42:18,497 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8] 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b2dbd64aca17411e850a19fcba3bfd5e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:18,497 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25bfb958de85478aacbd5072154c9e1d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3794ea35e44b5fb417bc59d3366f78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733294534362 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fbf46b5570d472e81a4d65ecb675d72, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733294534362 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a694a124c7f4a2faaebd062322b6c35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294534422 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 194437972e6b44ebad2c19a8e67fef1c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294534422 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8fa247474047bfa7216993b13fa69f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:18,498 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48aff79e7e44b4198fce3b5e72aafc8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:18,508 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#B#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:18,508 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/f7685b092d514389a35cd6ba171d1e7b is 50, key is test_row_0/B:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:18,509 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:18,511 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412046ba6546c99d443bcbef158556e0c7534_896711ea717d7f14b2e304328616f183 store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:18,514 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412046ba6546c99d443bcbef158556e0c7534_896711ea717d7f14b2e304328616f183, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:18,514 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046ba6546c99d443bcbef158556e0c7534_896711ea717d7f14b2e304328616f183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:18,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742215_1391 (size=12391) 2024-12-04T06:42:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742216_1392 (size=4469) 2024-12-04T06:42:18,520 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#A#compaction#336 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:18,520 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/dfecb133a25d41d09309676b5c2628be is 175, key is test_row_0/A:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742217_1393 (size=31345) 2024-12-04T06:42:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T06:42:18,677 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:18,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-04T06:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:18,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:18,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:18,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:18,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:18,833 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:18,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:18,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aa913408bfa6431aa83f03223abf41b5_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294536886/Put/seqid=0 2024-12-04T06:42:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742218_1394 (size=12304) 2024-12-04T06:42:18,920 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/f7685b092d514389a35cd6ba171d1e7b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/f7685b092d514389a35cd6ba171d1e7b 2024-12-04T06:42:18,924 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/B of 896711ea717d7f14b2e304328616f183 into f7685b092d514389a35cd6ba171d1e7b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:18,925 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:18,925 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/B, priority=12, startTime=1733294538496; duration=0sec 2024-12-04T06:42:18,925 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:18,925 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:B 2024-12-04T06:42:18,925 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:18,926 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:18,926 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/C is initiating minor compaction (all files) 2024-12-04T06:42:18,926 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/C in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:18,926 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0e2c6e67dbcb4e53a2297b62a0558a37, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=47.1 K 2024-12-04T06:42:18,926 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e2c6e67dbcb4e53a2297b62a0558a37, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733294533603 2024-12-04T06:42:18,926 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ec0079517b47e3b9da63068fc1f2da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733294534362 2024-12-04T06:42:18,927 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d157c4e5f176486ea8c1c16578358772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733294534422 2024-12-04T06:42:18,927 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c791cd6ece47ea8cde526874d33cda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:18,927 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/dfecb133a25d41d09309676b5c2628be as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be 2024-12-04T06:42:18,932 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/A of 896711ea717d7f14b2e304328616f183 into dfecb133a25d41d09309676b5c2628be(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:18,932 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:18,932 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/A, priority=12, startTime=1733294538495; duration=0sec 2024-12-04T06:42:18,933 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:18,933 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:A 2024-12-04T06:42:18,937 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:18,938 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4f8429db877a49148d60ae0aa1378f82 is 50, key is test_row_0/C:col10/1733294535731/Put/seqid=0 2024-12-04T06:42:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742219_1395 (size=12391) 2024-12-04T06:42:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:19,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294539179, max timestamp=1733294539196Num uniques 859; 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294539179, max timestamp=1733294539196Num uniques 859; 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294539179, max timestamp=1733294539196Num uniques 859; 2024-12-04T06:42:19,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:19,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:19,247 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aa913408bfa6431aa83f03223abf41b5_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa913408bfa6431aa83f03223abf41b5_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/cd2ea9775e1a46139fde5556f35e2c20, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:19,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/cd2ea9775e1a46139fde5556f35e2c20 is 175, key is test_row_0/A:col10/1733294536886/Put/seqid=0 2024-12-04T06:42:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742220_1396 (size=31105) 2024-12-04T06:42:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294599274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294599286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:19,348 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4f8429db877a49148d60ae0aa1378f82 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4f8429db877a49148d60ae0aa1378f82 2024-12-04T06:42:19,352 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/C of 896711ea717d7f14b2e304328616f183 into 4f8429db877a49148d60ae0aa1378f82(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:19,352 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:19,352 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/C, priority=12, startTime=1733294538496; duration=0sec 2024-12-04T06:42:19,352 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:19,352 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:C 2024-12-04T06:42:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294599396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294599396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294599616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294599616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,657 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/cd2ea9775e1a46139fde5556f35e2c20 2024-12-04T06:42:19,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/915b833f94fc4693aadede5ff67432b4 is 50, key is test_row_0/B:col10/1733294536886/Put/seqid=0 2024-12-04T06:42:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742221_1397 (size=12151) 2024-12-04T06:42:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:19,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294599937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294599937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:19,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294599976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:19,989 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,067 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/915b833f94fc4693aadede5ff67432b4 2024-12-04T06:42:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 is 50, key is test_row_0/C:col10/1733294536886/Put/seqid=0 2024-12-04T06:42:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742222_1398 (size=12151) 2024-12-04T06:42:20,078 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 2024-12-04T06:42:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/cd2ea9775e1a46139fde5556f35e2c20 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20 2024-12-04T06:42:20,085 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20, entries=150, sequenceid=144, filesize=30.4 K 2024-12-04T06:42:20,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/915b833f94fc4693aadede5ff67432b4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4 2024-12-04T06:42:20,089 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4, entries=150, sequenceid=144, filesize=11.9 K 2024-12-04T06:42:20,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 2024-12-04T06:42:20,094 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4, entries=150, sequenceid=144, filesize=11.9 K 2024-12-04T06:42:20,095 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 896711ea717d7f14b2e304328616f183 in 1262ms, sequenceid=144, compaction requested=false 2024-12-04T06:42:20,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:20,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:20,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T06:42:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T06:42:20,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T06:42:20,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-12-04T06:42:20,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4210 sec 2024-12-04T06:42:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:20,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:20,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:20,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e2f54ded2264b6f9bbba915c9e77ab3_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:20,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742223_1399 (size=14794) 2024-12-04T06:42:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294600535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294600536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294600657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294600657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294600716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,731 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T06:42:20,827 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:20,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-04T06:42:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:20,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:20,830 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:20,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:20,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294600877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294600877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:20,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:20,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045e2f54ded2264b6f9bbba915c9e77ab3_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e2f54ded2264b6f9bbba915c9e77ab3_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:20,885 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ca0b4cef2187438e8cc336ab57d2655b, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:20,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ca0b4cef2187438e8cc336ab57d2655b is 175, key is test_row_0/A:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:20,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742224_1400 (size=39749) 2024-12-04T06:42:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:20,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:20,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:20,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:21,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:21,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:21,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:21,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294601187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:21,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294601187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:21,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ca0b4cef2187438e8cc336ab57d2655b 2024-12-04T06:42:21,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/0a9dfdd5fce34f34aba928d1e065dbfb is 50, key is test_row_0/B:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:21,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742225_1401 (size=12151) 2024-12-04T06:42:21,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/0a9dfdd5fce34f34aba928d1e065dbfb 2024-12-04T06:42:21,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a545c671694b4d4482b26d8743e9f5af is 50, key is test_row_0/C:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742226_1402 (size=12151) 2024-12-04T06:42:21,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:21,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:21,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:21,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:21,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a545c671694b4d4482b26d8743e9f5af 2024-12-04T06:42:21,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294601707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:21,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294601707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ca0b4cef2187438e8cc336ab57d2655b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b 2024-12-04T06:42:21,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b, entries=200, sequenceid=183, filesize=38.8 K 2024-12-04T06:42:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/0a9dfdd5fce34f34aba928d1e065dbfb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb 2024-12-04T06:42:21,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb, entries=150, sequenceid=183, filesize=11.9 K 2024-12-04T06:42:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/a545c671694b4d4482b26d8743e9f5af as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af 2024-12-04T06:42:21,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af, entries=150, sequenceid=183, filesize=11.9 K 2024-12-04T06:42:21,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 896711ea717d7f14b2e304328616f183 in 1264ms, sequenceid=183, compaction requested=true 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:21,733 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:21,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:21,733 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/A is initiating minor compaction (all files) 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/B is initiating minor compaction (all files) 2024-12-04T06:42:21,734 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/B in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,734 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/A in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,734 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/f7685b092d514389a35cd6ba171d1e7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.8 K 2024-12-04T06:42:21,734 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=99.8 K 2024-12-04T06:42:21,734 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b] 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f7685b092d514389a35cd6ba171d1e7b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:21,734 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfecb133a25d41d09309676b5c2628be, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:21,735 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 915b833f94fc4693aadede5ff67432b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733294536886 2024-12-04T06:42:21,735 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd2ea9775e1a46139fde5556f35e2c20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733294536886 2024-12-04T06:42:21,735 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a9dfdd5fce34f34aba928d1e065dbfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:21,735 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0b4cef2187438e8cc336ab57d2655b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:21,746 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:21,746 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#B#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:21,747 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/5df97e61fa3c49789847f8ab3135f4a7 is 50, key is test_row_0/B:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:21,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-04T06:42:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:21,747 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:42:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:21,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:21,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:21,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:21,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:21,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:21,749 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412045948479f01074845b038c231130c1dc4_896711ea717d7f14b2e304328616f183 store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:21,751 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412045948479f01074845b038c231130c1dc4_896711ea717d7f14b2e304328616f183, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:21,751 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045948479f01074845b038c231130c1dc4_896711ea717d7f14b2e304328616f183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ddf0339302c44de098222e8f28bba3ca_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294540482/Put/seqid=0 2024-12-04T06:42:21,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742227_1403 (size=12493) 2024-12-04T06:42:21,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742228_1404 (size=4469) 2024-12-04T06:42:21,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742229_1405 (size=12304) 2024-12-04T06:42:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:21,778 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204ddf0339302c44de098222e8f28bba3ca_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddf0339302c44de098222e8f28bba3ca_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:21,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f43fa924788945cba5d06fc53f363243, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:21,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f43fa924788945cba5d06fc53f363243 is 175, key is test_row_0/A:col10/1733294540482/Put/seqid=0 2024-12-04T06:42:21,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742230_1406 (size=31105) 2024-12-04T06:42:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294541916, max timestamp=1733294542021Num uniques 859; 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294541916, max timestamp=1733294542021Num uniques 859; 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294541916, max timestamp=1733294542021Num uniques 859; 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:22,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:22,170 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/5df97e61fa3c49789847f8ab3135f4a7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5df97e61fa3c49789847f8ab3135f4a7 2024-12-04T06:42:22,170 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:22,171 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ff10af7a30fb4f379b77819625618812 is 175, key is test_row_0/A:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:22,178 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/B of 896711ea717d7f14b2e304328616f183 into 5df97e61fa3c49789847f8ab3135f4a7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:22,178 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:22,178 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/B, priority=13, startTime=1733294541733; duration=0sec 2024-12-04T06:42:22,178 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:22,178 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:B 2024-12-04T06:42:22,178 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:22,179 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:22,179 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/C is initiating minor compaction (all files) 2024-12-04T06:42:22,179 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/C in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:22,179 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4f8429db877a49148d60ae0aa1378f82, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.8 K 2024-12-04T06:42:22,180 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8429db877a49148d60ae0aa1378f82, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733294535731 2024-12-04T06:42:22,180 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7d94b43af84fd5aeef1fb88c4cd2c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733294536886 2024-12-04T06:42:22,180 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a545c671694b4d4482b26d8743e9f5af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:22,183 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f43fa924788945cba5d06fc53f363243 2024-12-04T06:42:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/40f52d96a2d94c5f9173f5d13e415f3c is 50, key is test_row_0/B:col10/1733294540482/Put/seqid=0 2024-12-04T06:42:22,192 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:22,192 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7b9ac3d5cd02453fb0cbaa927cc53c46 is 50, key is test_row_0/C:col10/1733294539256/Put/seqid=0 2024-12-04T06:42:22,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742231_1407 (size=31447) 2024-12-04T06:42:22,203 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/ff10af7a30fb4f379b77819625618812 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812 2024-12-04T06:42:22,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742232_1408 (size=12151) 2024-12-04T06:42:22,211 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/A of 896711ea717d7f14b2e304328616f183 into ff10af7a30fb4f379b77819625618812(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:22,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:22,211 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/A, priority=13, startTime=1733294541733; duration=0sec 2024-12-04T06:42:22,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:22,211 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:A 2024-12-04T06:42:22,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742233_1409 (size=12493) 2024-12-04T06:42:22,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294602242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294602346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294602557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,604 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/40f52d96a2d94c5f9173f5d13e415f3c 2024-12-04T06:42:22,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 is 50, key is test_row_0/C:col10/1733294540482/Put/seqid=0 2024-12-04T06:42:22,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742234_1410 (size=12151) 2024-12-04T06:42:22,615 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 2024-12-04T06:42:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f43fa924788945cba5d06fc53f363243 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243 2024-12-04T06:42:22,622 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243, entries=150, sequenceid=193, filesize=30.4 K 2024-12-04T06:42:22,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/40f52d96a2d94c5f9173f5d13e415f3c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c 2024-12-04T06:42:22,629 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c, entries=150, sequenceid=193, filesize=11.9 K 2024-12-04T06:42:22,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 2024-12-04T06:42:22,632 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7b9ac3d5cd02453fb0cbaa927cc53c46 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7b9ac3d5cd02453fb0cbaa927cc53c46 2024-12-04T06:42:22,633 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4, entries=150, sequenceid=193, filesize=11.9 K 2024-12-04T06:42:22,634 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 896711ea717d7f14b2e304328616f183 in 887ms, sequenceid=193, compaction requested=false 2024-12-04T06:42:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-04T06:42:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-04T06:42:22,636 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/C of 896711ea717d7f14b2e304328616f183 into 7b9ac3d5cd02453fb0cbaa927cc53c46(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:22,636 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:22,636 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/C, priority=13, startTime=1733294541733; duration=0sec 2024-12-04T06:42:22,636 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:22,636 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:C 2024-12-04T06:42:22,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-04T06:42:22,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-12-04T06:42:22,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8100 sec 2024-12-04T06:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:22,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:22,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120448ce26095ff049bd95357a6dc715e8cc_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742235_1411 (size=14794) 2024-12-04T06:42:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294602765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294602767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294602877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294602877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294602886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:22,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T06:42:22,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:22,977 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-04T06:42:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:22,979 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:22,980 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:22,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294603097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294603097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:23,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120448ce26095ff049bd95357a6dc715e8cc_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120448ce26095ff049bd95357a6dc715e8cc_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:23,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7a0543d7800b49889288a6883b74bfba, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:23,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7a0543d7800b49889288a6883b74bfba is 175, key is test_row_0/A:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:23,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742236_1412 (size=39749) 2024-12-04T06:42:23,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294603396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294603416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294603417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7a0543d7800b49889288a6883b74bfba 2024-12-04T06:42:23,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/bb610b381a81449e9c7c49fb86fa0d49 is 50, key is test_row_0/B:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:23,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742237_1413 (size=12151) 2024-12-04T06:42:23,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:23,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:23,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294603936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294603937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:23,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/bb610b381a81449e9c7c49fb86fa0d49 2024-12-04T06:42:23,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7e6239e4e5644524abc19ba730b0f40e is 50, key is test_row_0/C:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:23,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742238_1414 (size=12151) 2024-12-04T06:42:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60436 deadline: 1733294604026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:24,035 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:24,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:24,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7e6239e4e5644524abc19ba730b0f40e 2024-12-04T06:42:24,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7a0543d7800b49889288a6883b74bfba as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba 2024-12-04T06:42:24,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba, entries=200, sequenceid=233, filesize=38.8 K 2024-12-04T06:42:24,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/bb610b381a81449e9c7c49fb86fa0d49 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49 2024-12-04T06:42:24,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49, entries=150, sequenceid=233, filesize=11.9 K 2024-12-04T06:42:24,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7e6239e4e5644524abc19ba730b0f40e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e 2024-12-04T06:42:24,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e, entries=150, sequenceid=233, filesize=11.9 K 2024-12-04T06:42:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 896711ea717d7f14b2e304328616f183 in 1661ms, sequenceid=233, compaction requested=true 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:24,395 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:24,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:24,395 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,396 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:24,396 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:24,396 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/A is initiating minor compaction (all files) 2024-12-04T06:42:24,396 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/B is initiating minor compaction (all files) 2024-12-04T06:42:24,396 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/A in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,396 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/B in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,396 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5df97e61fa3c49789847f8ab3135f4a7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.9 K 2024-12-04T06:42:24,396 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=99.9 K 2024-12-04T06:42:24,396 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,397 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba] 2024-12-04T06:42:24,397 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff10af7a30fb4f379b77819625618812, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:24,397 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df97e61fa3c49789847f8ab3135f4a7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:24,397 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f52d96a2d94c5f9173f5d13e415f3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733294540482 2024-12-04T06:42:24,397 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f43fa924788945cba5d06fc53f363243, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733294540482 2024-12-04T06:42:24,398 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0543d7800b49889288a6883b74bfba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:24,398 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting bb610b381a81449e9c7c49fb86fa0d49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,405 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,406 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,407 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/05539d9564a8446bad343bcc14d1f59b is 50, key is test_row_0/B:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:24,407 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120444a19cf2eaa64496958cb8d93060d524_896711ea717d7f14b2e304328616f183 store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,409 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120444a19cf2eaa64496958cb8d93060d524_896711ea717d7f14b2e304328616f183, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:24,410 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120444a19cf2eaa64496958cb8d93060d524_896711ea717d7f14b2e304328616f183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742240_1416 (size=4469) 2024-12-04T06:42:24,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742239_1415 (size=12595) 2024-12-04T06:42:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#A#compaction#353 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,416 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f51d49b6fd3a4398a8c0cda146416419 is 175, key is test_row_0/A:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742241_1417 (size=31549) 2024-12-04T06:42:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/f51d49b6fd3a4398a8c0cda146416419 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,429 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/A of 896711ea717d7f14b2e304328616f183 into f51d49b6fd3a4398a8c0cda146416419(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:24,429 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:24,429 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/A, priority=13, startTime=1733294544395; duration=0sec 2024-12-04T06:42:24,429 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,429 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:A 2024-12-04T06:42:24,429 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,430 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:24,430 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/C is initiating minor compaction (all files) 2024-12-04T06:42:24,430 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/C in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,430 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7b9ac3d5cd02453fb0cbaa927cc53c46, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=35.9 K 2024-12-04T06:42:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,431 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9ac3d5cd02453fb0cbaa927cc53c46, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294539256 2024-12-04T06:42:24,431 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ccc02dae9f4c5d9cc7d37fbcbf6eb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733294540482 2024-12-04T06:42:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,431 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6239e4e5644524abc19ba730b0f40e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,437 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:24,438 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7f2b7315fa414dfca7dd6cfc13388707 is 50, key is test_row_0/C:col10/1733294542094/Put/seqid=0 2024-12-04T06:42:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742242_1418 (size=12595) 2024-12-04T06:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:24,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:42:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,447 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/7f2b7315fa414dfca7dd6cfc13388707 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7f2b7315fa414dfca7dd6cfc13388707 2024-12-04T06:42:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204598f99bb72414531a77baa896b31069a_896711ea717d7f14b2e304328616f183 is 50, key is test_row_1/A:col10/1733294542751/Put/seqid=0 2024-12-04T06:42:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,453 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/C of 896711ea717d7f14b2e304328616f183 into 7f2b7315fa414dfca7dd6cfc13388707(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:24,453 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:24,453 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/C, priority=13, startTime=1733294544395; duration=0sec 2024-12-04T06:42:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,453 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:24,454 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:C 2024-12-04T06:42:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742243_1419 (size=12304) 2024-12-04T06:42:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294544613, max timestamp=1733294544738Num uniques 859; 2024-12-04T06:42:24,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294544613, max timestamp=1733294544738Num uniques 859; 2024-12-04T06:42:24,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294544613, max timestamp=1733294544738Num uniques 859; 2024-12-04T06:42:24,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:24,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:24,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:42:24,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,821 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/05539d9564a8446bad343bcc14d1f59b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/05539d9564a8446bad343bcc14d1f59b 2024-12-04T06:42:24,825 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 896711ea717d7f14b2e304328616f183/B of 896711ea717d7f14b2e304328616f183 into 05539d9564a8446bad343bcc14d1f59b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:24,825 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:24,825 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/B, priority=13, startTime=1733294544395; duration=0sec 2024-12-04T06:42:24,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:24,826 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:B 2024-12-04T06:42:24,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:24,863 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204598f99bb72414531a77baa896b31069a_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204598f99bb72414531a77baa896b31069a_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:24,863 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/260a5570a4514c0396a7dc0606a13abf, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:24,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/260a5570a4514c0396a7dc0606a13abf is 175, key is test_row_1/A:col10/1733294542751/Put/seqid=0 2024-12-04T06:42:24,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742244_1420 (size=31101) 2024-12-04T06:42:24,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294604907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:24,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294604947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:24,958 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:24,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294604956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:24,966 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:24,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:24,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294605017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:25,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:25,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294605236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:25,268 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/260a5570a4514c0396a7dc0606a13abf 2024-12-04T06:42:25,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b18ccda32d1b4d07b029428bf1241ad6 is 50, key is test_row_1/B:col10/1733294542751/Put/seqid=0 2024-12-04T06:42:25,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742245_1421 (size=9757) 2024-12-04T06:42:25,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b18ccda32d1b4d07b029428bf1241ad6 2024-12-04T06:42:25,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/220829657a20482da53bffcff1820ba2 is 50, key is test_row_1/C:col10/1733294542751/Put/seqid=0 2024-12-04T06:42:25,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742246_1422 (size=9757) 2024-12-04T06:42:25,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294605547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:25,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:25,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:25,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/220829657a20482da53bffcff1820ba2 2024-12-04T06:42:25,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/260a5570a4514c0396a7dc0606a13abf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf 2024-12-04T06:42:25,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf, entries=150, sequenceid=244, filesize=30.4 K 2024-12-04T06:42:25,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b18ccda32d1b4d07b029428bf1241ad6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6 2024-12-04T06:42:25,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6, entries=100, sequenceid=244, filesize=9.5 K 2024-12-04T06:42:25,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/220829657a20482da53bffcff1820ba2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2 2024-12-04T06:42:25,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2, entries=100, sequenceid=244, filesize=9.5 K 2024-12-04T06:42:25,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 896711ea717d7f14b2e304328616f183 in 1261ms, sequenceid=244, compaction requested=false 2024-12-04T06:42:25,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:25,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-04T06:42:25,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:25,735 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:25,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aab1465346ac4ba8b431196a366b9475_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294544784/Put/seqid=0 2024-12-04T06:42:25,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742247_1423 (size=12454) 2024-12-04T06:42:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:26,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:26,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294606131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:26,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:26,150 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204aab1465346ac4ba8b431196a366b9475_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aab1465346ac4ba8b431196a366b9475_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/d2164f8d55804992891ed62dfae26201, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/d2164f8d55804992891ed62dfae26201 is 175, key is test_row_0/A:col10/1733294544784/Put/seqid=0 2024-12-04T06:42:26,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742248_1424 (size=31255) 2024-12-04T06:42:26,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294606256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294606477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:26,558 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/d2164f8d55804992891ed62dfae26201 2024-12-04T06:42:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ccd2ce2b6c3145b6a8340e094715867b is 50, key is test_row_0/B:col10/1733294544784/Put/seqid=0 2024-12-04T06:42:26,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742249_1425 (size=12301) 2024-12-04T06:42:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294606796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:26,972 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ccd2ce2b6c3145b6a8340e094715867b 2024-12-04T06:42:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9a317519a1fb45c193afc9ead33a2e4f is 50, key is test_row_0/C:col10/1733294544784/Put/seqid=0 2024-12-04T06:42:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742250_1426 (size=12301) 2024-12-04T06:42:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294606977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:26,987 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294606987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:27,001 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:27,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294607306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:27,385 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9a317519a1fb45c193afc9ead33a2e4f 2024-12-04T06:42:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/d2164f8d55804992891ed62dfae26201 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201 2024-12-04T06:42:27,392 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201, entries=150, sequenceid=282, filesize=30.5 K 2024-12-04T06:42:27,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/ccd2ce2b6c3145b6a8340e094715867b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b 2024-12-04T06:42:27,396 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T06:42:27,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/9a317519a1fb45c193afc9ead33a2e4f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f 2024-12-04T06:42:27,400 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f, entries=150, sequenceid=282, filesize=12.0 K 2024-12-04T06:42:27,401 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 896711ea717d7f14b2e304328616f183 in 1666ms, sequenceid=282, compaction requested=true 2024-12-04T06:42:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-04T06:42:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-04T06:42:27,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-04T06:42:27,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4220 sec 2024-12-04T06:42:27,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 4.4270 sec 2024-12-04T06:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:28,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:28,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:28,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120434fd14d15c2745a188c9fe47d6f671c9_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742251_1427 (size=12454) 2024-12-04T06:42:28,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294548392, max timestamp=1733294548507Num uniques 859; 2024-12-04T06:42:28,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294548392, max timestamp=1733294548507Num uniques 859; 2024-12-04T06:42:28,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:28,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294548392, max timestamp=1733294548507Num uniques 859; 2024-12-04T06:42:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294608678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:28,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:28,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120434fd14d15c2745a188c9fe47d6f671c9_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd14d15c2745a188c9fe47d6f671c9_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:28,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7065f96fcb96445d98ae46eb5b25fa3a, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:28,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7065f96fcb96445d98ae46eb5b25fa3a is 175, key is test_row_0/A:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742252_1428 (size=31255) 2024-12-04T06:42:28,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294608797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:29,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294609016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:29,081 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:42:29,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7065f96fcb96445d98ae46eb5b25fa3a 2024-12-04T06:42:29,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b761f245f03a40c98c623067ca92fa51 is 50, key is test_row_0/B:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:29,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742253_1429 (size=12301) 2024-12-04T06:42:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294609337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:29,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b761f245f03a40c98c623067ca92fa51 2024-12-04T06:42:29,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4ab72491bbac4db6a92ee8435c8a6760 is 50, key is test_row_0/C:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:29,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742254_1430 (size=12301) 2024-12-04T06:42:29,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294609856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:29,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:29,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4ab72491bbac4db6a92ee8435c8a6760 2024-12-04T06:42:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/7065f96fcb96445d98ae46eb5b25fa3a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a 2024-12-04T06:42:29,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a, entries=150, sequenceid=293, filesize=30.5 K 2024-12-04T06:42:29,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b761f245f03a40c98c623067ca92fa51 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51 2024-12-04T06:42:29,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T06:42:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4ab72491bbac4db6a92ee8435c8a6760 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760 2024-12-04T06:42:29,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T06:42:29,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 896711ea717d7f14b2e304328616f183 in 1660ms, sequenceid=293, compaction requested=true 2024-12-04T06:42:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:29,993 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:29,993 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 896711ea717d7f14b2e304328616f183:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:29,993 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125160 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:29,993 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/B is initiating minor compaction (all files) 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/A is initiating minor compaction (all files) 2024-12-04T06:42:29,994 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/B in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:29,994 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/A in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:29,994 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/05539d9564a8446bad343bcc14d1f59b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=45.9 K 2024-12-04T06:42:29,994 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=122.2 K 2024-12-04T06:42:29,994 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a] 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 05539d9564a8446bad343bcc14d1f59b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f51d49b6fd3a4398a8c0cda146416419, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:29,994 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b18ccda32d1b4d07b029428bf1241ad6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294542751 2024-12-04T06:42:29,995 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260a5570a4514c0396a7dc0606a13abf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294542751 2024-12-04T06:42:29,995 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd2ce2b6c3145b6a8340e094715867b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733294544784 2024-12-04T06:42:29,995 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2164f8d55804992891ed62dfae26201, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733294544784 2024-12-04T06:42:29,995 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b761f245f03a40c98c623067ca92fa51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733294546118 2024-12-04T06:42:29,995 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7065f96fcb96445d98ae46eb5b25fa3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733294546118 2024-12-04T06:42:30,001 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:30,003 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:30,003 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120440cc525d30b7419b93f698375306abb3_896711ea717d7f14b2e304328616f183 store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:30,003 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/e11fc4e059c14069ad6fd4da38a133fa is 50, key is test_row_0/B:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:30,005 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120440cc525d30b7419b93f698375306abb3_896711ea717d7f14b2e304328616f183, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:30,005 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120440cc525d30b7419b93f698375306abb3_896711ea717d7f14b2e304328616f183 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:30,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742255_1431 (size=12881) 2024-12-04T06:42:30,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742256_1432 (size=4469) 2024-12-04T06:42:30,410 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:30,410 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/4b151e1b1f664c429e9c54d3b8309eea is 175, key is test_row_0/A:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:30,414 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/e11fc4e059c14069ad6fd4da38a133fa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/e11fc4e059c14069ad6fd4da38a133fa 2024-12-04T06:42:30,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742257_1433 (size=31835) 2024-12-04T06:42:30,418 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/4b151e1b1f664c429e9c54d3b8309eea as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/4b151e1b1f664c429e9c54d3b8309eea 2024-12-04T06:42:30,419 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/B of 896711ea717d7f14b2e304328616f183 into e11fc4e059c14069ad6fd4da38a133fa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:30,419 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:30,419 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/B, priority=12, startTime=1733294549993; duration=0sec 2024-12-04T06:42:30,419 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:30,419 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:B 2024-12-04T06:42:30,419 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:30,420 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:30,420 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 896711ea717d7f14b2e304328616f183/C is initiating minor compaction (all files) 2024-12-04T06:42:30,420 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 896711ea717d7f14b2e304328616f183/C in TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:30,420 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7f2b7315fa414dfca7dd6cfc13388707, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp, totalSize=45.9 K 2024-12-04T06:42:30,420 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2b7315fa414dfca7dd6cfc13388707, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733294542094 2024-12-04T06:42:30,421 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 220829657a20482da53bffcff1820ba2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733294542751 2024-12-04T06:42:30,421 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a317519a1fb45c193afc9ead33a2e4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733294544784 2024-12-04T06:42:30,422 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab72491bbac4db6a92ee8435c8a6760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733294546118 2024-12-04T06:42:30,423 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/A of 896711ea717d7f14b2e304328616f183 into 4b151e1b1f664c429e9c54d3b8309eea(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:30,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:30,424 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/A, priority=12, startTime=1733294549992; duration=0sec 2024-12-04T06:42:30,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:30,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:A 2024-12-04T06:42:30,432 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 896711ea717d7f14b2e304328616f183#C#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:30,432 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4b3df988dbb846fbaae23294f2450d4c is 50, key is test_row_0/C:col10/1733294548331/Put/seqid=0 2024-12-04T06:42:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742258_1434 (size=12881) 2024-12-04T06:42:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:30,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047fb35cd1cc4b44aa82b20a34fac63a73_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294548676/Put/seqid=0 2024-12-04T06:42:30,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742259_1435 (size=14994) 2024-12-04T06:42:30,792 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:30,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412047fb35cd1cc4b44aa82b20a34fac63a73_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047fb35cd1cc4b44aa82b20a34fac63a73_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:30,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5f8ccbc930114b9dbccea9ecc29322af, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:30,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5f8ccbc930114b9dbccea9ecc29322af is 175, key is test_row_0/A:col10/1733294548676/Put/seqid=0 2024-12-04T06:42:30,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742260_1436 (size=39949) 2024-12-04T06:42:30,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5f8ccbc930114b9dbccea9ecc29322af 2024-12-04T06:42:30,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/5a23bdcd49e641a388d7de3e83c64c78 is 50, key is test_row_0/B:col10/1733294548676/Put/seqid=0 2024-12-04T06:42:30,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742261_1437 (size=12301) 2024-12-04T06:42:30,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/5a23bdcd49e641a388d7de3e83c64c78 2024-12-04T06:42:30,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/93d479145596469ea2ce285a268c6b5d is 50, key is test_row_0/C:col10/1733294548676/Put/seqid=0 2024-12-04T06:42:30,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742262_1438 (size=12301) 2024-12-04T06:42:30,840 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/4b3df988dbb846fbaae23294f2450d4c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4b3df988dbb846fbaae23294f2450d4c 2024-12-04T06:42:30,844 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 896711ea717d7f14b2e304328616f183/C of 896711ea717d7f14b2e304328616f183 into 4b3df988dbb846fbaae23294f2450d4c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:30,844 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:30,844 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., storeName=896711ea717d7f14b2e304328616f183/C, priority=12, startTime=1733294549993; duration=0sec 2024-12-04T06:42:30,845 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:30,845 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 896711ea717d7f14b2e304328616f183:C 2024-12-04T06:42:30,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294610850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60378 deadline: 1733294610877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:30,886 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294610956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:31,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60408 deadline: 1733294611016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:31,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60406 deadline: 1733294611026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:31,030 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,030 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 896711ea717d7f14b2e304328616f183 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60384 deadline: 1733294611177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:42:31,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T06:42:31,187 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:31,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-04T06:42:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T06:42:31,189 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:31,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:31,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:31,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/93d479145596469ea2ce285a268c6b5d 2024-12-04T06:42:31,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5f8ccbc930114b9dbccea9ecc29322af as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5f8ccbc930114b9dbccea9ecc29322af 2024-12-04T06:42:31,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5f8ccbc930114b9dbccea9ecc29322af, entries=200, sequenceid=332, filesize=39.0 K 2024-12-04T06:42:31,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/5a23bdcd49e641a388d7de3e83c64c78 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5a23bdcd49e641a388d7de3e83c64c78 2024-12-04T06:42:31,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5a23bdcd49e641a388d7de3e83c64c78, entries=150, sequenceid=332, filesize=12.0 K 2024-12-04T06:42:31,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/93d479145596469ea2ce285a268c6b5d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/93d479145596469ea2ce285a268c6b5d 2024-12-04T06:42:31,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/93d479145596469ea2ce285a268c6b5d, entries=150, sequenceid=332, filesize=12.0 K 2024-12-04T06:42:31,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 896711ea717d7f14b2e304328616f183 in 460ms, sequenceid=332, compaction requested=false 2024-12-04T06:42:31,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T06:42:31,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-04T06:42:31,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:31,342 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cd2875e24b3e46d08267b832c2da502f_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294550833/Put/seqid=0 2024-12-04T06:42:31,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742263_1439 (size=12454) 2024-12-04T06:42:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,356 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204cd2875e24b3e46d08267b832c2da502f_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cd2875e24b3e46d08267b832c2da502f_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:31,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/9eaeb59bd9cf4b79aaa70434197fda32, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/9eaeb59bd9cf4b79aaa70434197fda32 is 175, key is test_row_0/A:col10/1733294550833/Put/seqid=0 2024-12-04T06:42:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742264_1440 (size=31255) 2024-12-04T06:42:31,365 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/9eaeb59bd9cf4b79aaa70434197fda32 2024-12-04T06:42:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/d300fd8030834c9689ac73354ae42ffa is 50, key is test_row_0/B:col10/1733294550833/Put/seqid=0 2024-12-04T06:42:31,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742265_1441 (size=12301) 2024-12-04T06:42:31,375 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/d300fd8030834c9689ac73354ae42ffa 2024-12-04T06:42:31,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3358ff04ddc8421d8416a28429970e3f is 50, key is test_row_0/C:col10/1733294550833/Put/seqid=0 2024-12-04T06:42:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742266_1442 (size=12301) 2024-12-04T06:42:31,385 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3358ff04ddc8421d8416a28429970e3f 2024-12-04T06:42:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/9eaeb59bd9cf4b79aaa70434197fda32 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/9eaeb59bd9cf4b79aaa70434197fda32 2024-12-04T06:42:31,392 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/9eaeb59bd9cf4b79aaa70434197fda32, entries=150, sequenceid=343, filesize=30.5 K 2024-12-04T06:42:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/d300fd8030834c9689ac73354ae42ffa as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/d300fd8030834c9689ac73354ae42ffa 2024-12-04T06:42:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,399 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/d300fd8030834c9689ac73354ae42ffa, entries=150, sequenceid=343, filesize=12.0 K 2024-12-04T06:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/3358ff04ddc8421d8416a28429970e3f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3358ff04ddc8421d8416a28429970e3f 2024-12-04T06:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,404 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3358ff04ddc8421d8416a28429970e3f, entries=150, sequenceid=343, filesize=12.0 K 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,405 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 896711ea717d7f14b2e304328616f183 in 63ms, sequenceid=343, compaction requested=true 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:31,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-04T06:42:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-04T06:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-04T06:42:31,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 217 msec 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 221 msec 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T06:42:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-04T06:42:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,511 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:31,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-04T06:42:31,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,665 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T06:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:31,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120406337c25e9af4d5bb6956a8b7bffed26_896711ea717d7f14b2e304328616f183 is 50, key is test_row_1/A:col10/1733294551581/Put/seqid=0 2024-12-04T06:42:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742267_1443 (size=9914) 2024-12-04T06:42:31,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:31,685 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120406337c25e9af4d5bb6956a8b7bffed26_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406337c25e9af4d5bb6956a8b7bffed26_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:31,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/b45114334a83418fb12af637a4e37fc1, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:31,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/b45114334a83418fb12af637a4e37fc1 is 175, key is test_row_1/A:col10/1733294551581/Put/seqid=0 2024-12-04T06:42:31,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742268_1444 (size=22561) 2024-12-04T06:42:31,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. as already flushing 2024-12-04T06:42:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:31,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294551773, max timestamp=1733294551799Num uniques 859; 2024-12-04T06:42:31,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:31,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294551773, max timestamp=1733294551799Num uniques 859; 2024-12-04T06:42:31,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:31,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294551773, max timestamp=1733294551799Num uniques 859; 2024-12-04T06:42:31,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:42:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:31,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733294551910, max timestamp=1733294551976Num uniques 1018; 2024-12-04T06:42:31,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733294551910, max timestamp=1733294551976Num uniques 1018; 2024-12-04T06:42:31,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T06:42:31,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T06:42:31,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733294551910, max timestamp=1733294551976Num uniques 1018; 2024-12-04T06:42:31,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-04T06:42:32,094 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/b45114334a83418fb12af637a4e37fc1 2024-12-04T06:42:32,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/6f8e21e4f0534a0aba43e390ef60312a is 50, key is test_row_1/B:col10/1733294551581/Put/seqid=0 2024-12-04T06:42:32,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742269_1445 (size=9857) 2024-12-04T06:42:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733294552108, max timestamp=1733294552138Num uniques 1027; 2024-12-04T06:42:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T06:42:32,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733294552108, max timestamp=1733294552138Num uniques 1027; 2024-12-04T06:42:32,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T06:42:32,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733294552108, max timestamp=1733294552138Num uniques 1027; 2024-12-04T06:42:32,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-04T06:42:32,361 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-12-04T06:42:32,362 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,362 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,362 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,363 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-12-04T06:42:32,363 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,363 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,363 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,364 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-12-04T06:42:32,364 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,364 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,364 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,371 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-12-04T06:42:32,371 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,371 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,371 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,376 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-12-04T06:42:32,376 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,376 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,376 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,382 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-12-04T06:42:32,382 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,382 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,382 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,506 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/6f8e21e4f0534a0aba43e390ef60312a 2024-12-04T06:42:32,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/6ca26d35d6e74d238e0167148e266596 is 50, key is test_row_1/C:col10/1733294551581/Put/seqid=0 2024-12-04T06:42:32,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742270_1446 (size=9857) 2024-12-04T06:42:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:32,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a] to archive 2024-12-04T06:42:32,759 WARN [regionserver/607fd5c6574c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-04T06:42:32,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:32,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/e65a6c02c82849918c3a7eb2db97dbb3 2024-12-04T06:42:32,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ee625d7480814982a2005d77a6a853d2 2024-12-04T06:42:32,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/0235a9e24e744c0d91c905f089db646c 2024-12-04T06:42:32,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/25bfb958de85478aacbd5072154c9e1d 2024-12-04T06:42:32,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7fbf46b5570d472e81a4d65ecb675d72 2024-12-04T06:42:32,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/194437972e6b44ebad2c19a8e67fef1c 2024-12-04T06:42:32,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/a48aff79e7e44b4198fce3b5e72aafc8 2024-12-04T06:42:32,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/dfecb133a25d41d09309676b5c2628be 2024-12-04T06:42:32,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/cd2ea9775e1a46139fde5556f35e2c20 2024-12-04T06:42:32,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ca0b4cef2187438e8cc336ab57d2655b 2024-12-04T06:42:32,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/ff10af7a30fb4f379b77819625618812 2024-12-04T06:42:32,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f43fa924788945cba5d06fc53f363243 2024-12-04T06:42:32,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7a0543d7800b49889288a6883b74bfba 2024-12-04T06:42:32,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/f51d49b6fd3a4398a8c0cda146416419 2024-12-04T06:42:32,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/260a5570a4514c0396a7dc0606a13abf 2024-12-04T06:42:32,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/d2164f8d55804992891ed62dfae26201 2024-12-04T06:42:32,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/7065f96fcb96445d98ae46eb5b25fa3a 2024-12-04T06:42:32,778 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b2dbd64aca17411e850a19fcba3bfd5e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/f7685b092d514389a35cd6ba171d1e7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5df97e61fa3c49789847f8ab3135f4a7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/05539d9564a8446bad343bcc14d1f59b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51] to archive 2024-12-04T06:42:32,778 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:32,779 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/9f351a9327164a4faefd77e54972bff7 2024-12-04T06:42:32,780 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/de87aee153074abb963b699137649819 2024-12-04T06:42:32,781 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b2dbd64aca17411e850a19fcba3bfd5e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b2dbd64aca17411e850a19fcba3bfd5e 2024-12-04T06:42:32,782 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/56e19e88421749b9adb106ea97afdf52 2024-12-04T06:42:32,783 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/1c3794ea35e44b5fb417bc59d3366f78 2024-12-04T06:42:32,784 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/4a694a124c7f4a2faaebd062322b6c35 2024-12-04T06:42:32,785 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/f7685b092d514389a35cd6ba171d1e7b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/f7685b092d514389a35cd6ba171d1e7b 2024-12-04T06:42:32,786 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ab8fa247474047bfa7216993b13fa69f 2024-12-04T06:42:32,787 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/915b833f94fc4693aadede5ff67432b4 2024-12-04T06:42:32,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5df97e61fa3c49789847f8ab3135f4a7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5df97e61fa3c49789847f8ab3135f4a7 2024-12-04T06:42:32,788 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/0a9dfdd5fce34f34aba928d1e065dbfb 2024-12-04T06:42:32,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/40f52d96a2d94c5f9173f5d13e415f3c 2024-12-04T06:42:32,790 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/05539d9564a8446bad343bcc14d1f59b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/05539d9564a8446bad343bcc14d1f59b 2024-12-04T06:42:32,791 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/bb610b381a81449e9c7c49fb86fa0d49 2024-12-04T06:42:32,792 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b18ccda32d1b4d07b029428bf1241ad6 2024-12-04T06:42:32,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/ccd2ce2b6c3145b6a8340e094715867b 2024-12-04T06:42:32,794 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b761f245f03a40c98c623067ca92fa51 2024-12-04T06:42:32,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0e2c6e67dbcb4e53a2297b62a0558a37, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4f8429db877a49148d60ae0aa1378f82, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7b9ac3d5cd02453fb0cbaa927cc53c46, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7f2b7315fa414dfca7dd6cfc13388707, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760] to archive 2024-12-04T06:42:32,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:42:32,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3f23d48c6f384f3f9353d320446af48d 2024-12-04T06:42:32,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/b83c37f676a249d7b3c2f635d1e4a673 2024-12-04T06:42:32,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0e2c6e67dbcb4e53a2297b62a0558a37 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0e2c6e67dbcb4e53a2297b62a0558a37 2024-12-04T06:42:32,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9e8b2992093e470ebf25d3368a74f394 2024-12-04T06:42:32,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a7ec0079517b47e3b9da63068fc1f2da 2024-12-04T06:42:32,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/d157c4e5f176486ea8c1c16578358772 2024-12-04T06:42:32,803 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4f8429db877a49148d60ae0aa1378f82 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4f8429db877a49148d60ae0aa1378f82 2024-12-04T06:42:32,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/92c791cd6ece47ea8cde526874d33cda 2024-12-04T06:42:32,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/0c7d94b43af84fd5aeef1fb88c4cd2c4 2024-12-04T06:42:32,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7b9ac3d5cd02453fb0cbaa927cc53c46 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7b9ac3d5cd02453fb0cbaa927cc53c46 2024-12-04T06:42:32,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/a545c671694b4d4482b26d8743e9f5af 2024-12-04T06:42:32,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/88ccc02dae9f4c5d9cc7d37fbcbf6eb4 2024-12-04T06:42:32,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7f2b7315fa414dfca7dd6cfc13388707 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7f2b7315fa414dfca7dd6cfc13388707 2024-12-04T06:42:32,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/7e6239e4e5644524abc19ba730b0f40e 2024-12-04T06:42:32,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/220829657a20482da53bffcff1820ba2 2024-12-04T06:42:32,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/9a317519a1fb45c193afc9ead33a2e4f 2024-12-04T06:42:32,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/607fd5c6574c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4ab72491bbac4db6a92ee8435c8a6760 2024-12-04T06:42:32,898 INFO [Thread-1647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1647. 2024-12-04T06:42:32,898 DEBUG [Thread-1647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:32,898 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,898 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:32,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:32,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733294552243, max timestamp=1733294552897Num uniques 1036; 2024-12-04T06:42:32,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-04T06:42:32,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733294552243, max timestamp=1733294552897Num uniques 1036; 2024-12-04T06:42:32,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733294552243, max timestamp=1733294552897Num uniques 1036; 2024-12-04T06:42:32,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-04T06:42:32,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-04T06:42:32,915 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/6ca26d35d6e74d238e0167148e266596 2024-12-04T06:42:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/b45114334a83418fb12af637a4e37fc1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/b45114334a83418fb12af637a4e37fc1 2024-12-04T06:42:32,921 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/b45114334a83418fb12af637a4e37fc1, entries=100, sequenceid=348, filesize=22.0 K 2024-12-04T06:42:32,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/6f8e21e4f0534a0aba43e390ef60312a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/6f8e21e4f0534a0aba43e390ef60312a 2024-12-04T06:42:32,925 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/6f8e21e4f0534a0aba43e390ef60312a, entries=100, sequenceid=348, filesize=9.6 K 2024-12-04T06:42:32,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/6ca26d35d6e74d238e0167148e266596 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/6ca26d35d6e74d238e0167148e266596 2024-12-04T06:42:32,928 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/6ca26d35d6e74d238e0167148e266596, entries=100, sequenceid=348, filesize=9.6 K 2024-12-04T06:42:32,929 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=25.23 KB/25834 for 896711ea717d7f14b2e304328616f183 in 1265ms, sequenceid=348, compaction requested=true 2024-12-04T06:42:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 896711ea717d7f14b2e304328616f183: 2024-12-04T06:42:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-04T06:42:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-04T06:42:32,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-04T06:42:32,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-12-04T06:42:32,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4240 sec 2024-12-04T06:42:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-04T06:42:33,657 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:34,078 INFO [Thread-1645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1645. 2024-12-04T06:42:34,078 DEBUG [Thread-1645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:34,078 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:34,078 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:34,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:41,088 INFO [Thread-1641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1641. 2024-12-04T06:42:41,088 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-12-04T06:42:41,088 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:41,088 DEBUG [Thread-1641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:42:41,088 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,088 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,088 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,088 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:41,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 909 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2721 rows 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 909 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2727 rows 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 905 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2714 rows 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 904 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2711 rows 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 907 2024-12-04T06:42:41,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2719 rows 2024-12-04T06:42:41,089 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:42:41,089 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:42:41,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:41,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:42:41,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:42:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:41,092 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294561092"}]},"ts":"1733294561092"} 2024-12-04T06:42:41,093 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:42:41,093 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:42:41,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:42:41,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, UNASSIGN}] 2024-12-04T06:42:41,095 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, UNASSIGN 2024-12-04T06:42:41,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:41,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, UNASSIGN because future has completed 2024-12-04T06:42:41,097 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:42:41,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:41,249 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 896711ea717d7f14b2e304328616f183, disabling compactions & flushes 2024-12-04T06:42:41,249 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. after waiting 0 ms 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:41,249 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 896711ea717d7f14b2e304328616f183 3/3 column families, dataSize=45.36 KB heapSize=119.81 KB 2024-12-04T06:42:41,249 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=A 2024-12-04T06:42:41,250 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:41,250 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=B 2024-12-04T06:42:41,250 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:41,250 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 896711ea717d7f14b2e304328616f183, store=C 2024-12-04T06:42:41,250 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:41,254 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046a9eca4fac9d46b0a38129800fefa329_896711ea717d7f14b2e304328616f183 is 50, key is test_row_0/A:col10/1733294561087/Put/seqid=0 2024-12-04T06:42:41,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742271_1447 (size=12454) 2024-12-04T06:42:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:41,658 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:41,661 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412046a9eca4fac9d46b0a38129800fefa329_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046a9eca4fac9d46b0a38129800fefa329_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:41,662 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5926ad8d7b094d028307ad2123490df0, store: [table=TestAcidGuarantees family=A region=896711ea717d7f14b2e304328616f183] 2024-12-04T06:42:41,662 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5926ad8d7b094d028307ad2123490df0 is 175, key is test_row_0/A:col10/1733294561087/Put/seqid=0 2024-12-04T06:42:41,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742272_1448 (size=31255) 2024-12-04T06:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:42,066 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5926ad8d7b094d028307ad2123490df0 2024-12-04T06:42:42,072 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b19cabed0433487291eedfb650c049a6 is 50, key is test_row_0/B:col10/1733294561087/Put/seqid=0 2024-12-04T06:42:42,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742273_1449 (size=12301) 2024-12-04T06:42:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:42,475 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b19cabed0433487291eedfb650c049a6 2024-12-04T06:42:42,480 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/838df346942d40e9ab632b86629a6ca1 is 50, key is test_row_0/C:col10/1733294561087/Put/seqid=0 2024-12-04T06:42:42,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742274_1450 (size=12301) 2024-12-04T06:42:42,884 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.05 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/838df346942d40e9ab632b86629a6ca1 2024-12-04T06:42:42,887 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/A/5926ad8d7b094d028307ad2123490df0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5926ad8d7b094d028307ad2123490df0 2024-12-04T06:42:42,890 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5926ad8d7b094d028307ad2123490df0, entries=150, sequenceid=423, filesize=30.5 K 2024-12-04T06:42:42,891 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/B/b19cabed0433487291eedfb650c049a6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b19cabed0433487291eedfb650c049a6 2024-12-04T06:42:42,894 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b19cabed0433487291eedfb650c049a6, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T06:42:42,894 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/.tmp/C/838df346942d40e9ab632b86629a6ca1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/838df346942d40e9ab632b86629a6ca1 2024-12-04T06:42:42,897 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/838df346942d40e9ab632b86629a6ca1, entries=150, sequenceid=423, filesize=12.0 K 2024-12-04T06:42:42,898 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~45.36 KB/46444, heapSize ~119.77 KB/122640, currentSize=0 B/0 for 896711ea717d7f14b2e304328616f183 in 1649ms, sequenceid=423, compaction requested=true 2024-12-04T06:42:42,901 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-12-04T06:42:42,902 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. 2024-12-04T06:42:42,902 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 896711ea717d7f14b2e304328616f183: Waiting for close lock at 1733294561249Running coprocessor pre-close hooks at 1733294561249Disabling compacts and flushes for region at 1733294561249Disabling writes for close at 1733294561249Obtaining lock to block concurrent updates at 1733294561249Preparing flush snapshotting stores in 896711ea717d7f14b2e304328616f183 at 1733294561249Finished memstore snapshotting TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183., syncing WAL and waiting on mvcc, flushsize=dataSize=46444, getHeapSize=122640, getOffHeapSize=0, getCellsCount=1014 at 1733294561250 (+1 ms)Flushing stores of TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183. at 1733294561250Flushing 896711ea717d7f14b2e304328616f183/A: creating writer at 1733294561250Flushing mob file 896711ea717d7f14b2e304328616f183/A: appending metadata at 1733294561254 (+4 ms)Flushing mob file 896711ea717d7f14b2e304328616f183/A: closing flushed file at 1733294561254Flushing 896711ea717d7f14b2e304328616f183/A: appending metadata at 1733294561662 (+408 ms)Flushing 896711ea717d7f14b2e304328616f183/A: closing flushed file at 1733294561662Flushing 896711ea717d7f14b2e304328616f183/B: creating writer at 1733294562069 (+407 ms)Flushing 896711ea717d7f14b2e304328616f183/B: appending metadata at 1733294562071 (+2 ms)Flushing 896711ea717d7f14b2e304328616f183/B: closing flushed file at 1733294562071Flushing 896711ea717d7f14b2e304328616f183/C: creating writer at 1733294562478 (+407 ms)Flushing 896711ea717d7f14b2e304328616f183/C: appending metadata at 1733294562480 (+2 ms)Flushing 896711ea717d7f14b2e304328616f183/C: closing flushed file at 1733294562480Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364b16c5: reopening flushed file at 1733294562887 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4fc967: reopening flushed file at 1733294562890 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c0da1: reopening flushed file at 1733294562894 (+4 ms)Finished flush of dataSize ~45.36 KB/46444, heapSize ~119.77 KB/122640, currentSize=0 B/0 for 896711ea717d7f14b2e304328616f183 in 1649ms, sequenceid=423, compaction requested=true at 1733294562898 (+4 ms)Writing region close event to WAL at 1733294562899 (+1 ms)Running coprocessor post-close hooks at 1733294562902 (+3 ms)Closed at 1733294562902 2024-12-04T06:42:42,904 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 896711ea717d7f14b2e304328616f183 2024-12-04T06:42:42,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=896711ea717d7f14b2e304328616f183, regionState=CLOSED 2024-12-04T06:42:42,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:42,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-04T06:42:42,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 896711ea717d7f14b2e304328616f183, server=607fd5c6574c,38257,1733294430862 in 1.8090 sec 2024-12-04T06:42:42,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-04T06:42:42,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=896711ea717d7f14b2e304328616f183, UNASSIGN in 1.8120 sec 2024-12-04T06:42:42,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-04T06:42:42,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8170 sec 2024-12-04T06:42:42,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294562912"}]},"ts":"1733294562912"} 2024-12-04T06:42:42,913 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:42:42,913 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:42:42,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8240 sec 2024-12-04T06:42:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-04T06:42:43,237 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:43,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:42:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,240 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T06:42:43,240 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,242 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,244 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits] 2024-12-04T06:42:43,246 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/4b151e1b1f664c429e9c54d3b8309eea to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/4b151e1b1f664c429e9c54d3b8309eea 2024-12-04T06:42:43,247 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5926ad8d7b094d028307ad2123490df0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5926ad8d7b094d028307ad2123490df0 2024-12-04T06:42:43,247 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5f8ccbc930114b9dbccea9ecc29322af to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/5f8ccbc930114b9dbccea9ecc29322af 2024-12-04T06:42:43,248 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/9eaeb59bd9cf4b79aaa70434197fda32 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/9eaeb59bd9cf4b79aaa70434197fda32 2024-12-04T06:42:43,249 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/b45114334a83418fb12af637a4e37fc1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/A/b45114334a83418fb12af637a4e37fc1 2024-12-04T06:42:43,251 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5a23bdcd49e641a388d7de3e83c64c78 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/5a23bdcd49e641a388d7de3e83c64c78 2024-12-04T06:42:43,252 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/6f8e21e4f0534a0aba43e390ef60312a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/6f8e21e4f0534a0aba43e390ef60312a 2024-12-04T06:42:43,253 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b19cabed0433487291eedfb650c049a6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/b19cabed0433487291eedfb650c049a6 2024-12-04T06:42:43,254 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/d300fd8030834c9689ac73354ae42ffa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/d300fd8030834c9689ac73354ae42ffa 2024-12-04T06:42:43,255 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/e11fc4e059c14069ad6fd4da38a133fa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/B/e11fc4e059c14069ad6fd4da38a133fa 2024-12-04T06:42:43,256 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3358ff04ddc8421d8416a28429970e3f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/3358ff04ddc8421d8416a28429970e3f 2024-12-04T06:42:43,257 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4b3df988dbb846fbaae23294f2450d4c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/4b3df988dbb846fbaae23294f2450d4c 2024-12-04T06:42:43,258 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/6ca26d35d6e74d238e0167148e266596 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/6ca26d35d6e74d238e0167148e266596 2024-12-04T06:42:43,259 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/838df346942d40e9ab632b86629a6ca1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/838df346942d40e9ab632b86629a6ca1 2024-12-04T06:42:43,259 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/93d479145596469ea2ce285a268c6b5d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/C/93d479145596469ea2ce285a268c6b5d 2024-12-04T06:42:43,261 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits/426.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183/recovered.edits/426.seqid 2024-12-04T06:42:43,261 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,262 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:42:43,262 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:42:43,262 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T06:42:43,264 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406337c25e9af4d5bb6956a8b7bffed26_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120406337c25e9af4d5bb6956a8b7bffed26_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,265 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120408f9e7de8ea0436caff431940ba7c7a6_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120408f9e7de8ea0436caff431940ba7c7a6_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,266 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd14d15c2745a188c9fe47d6f671c9_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd14d15c2745a188c9fe47d6f671c9_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,267 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd33389129466286ba3a5bf9055578_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120434fd33389129466286ba3a5bf9055578_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,268 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120448ce26095ff049bd95357a6dc715e8cc_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120448ce26095ff049bd95357a6dc715e8cc_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,269 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204598f99bb72414531a77baa896b31069a_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204598f99bb72414531a77baa896b31069a_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,269 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e2f54ded2264b6f9bbba915c9e77ab3_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045e2f54ded2264b6f9bbba915c9e77ab3_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,270 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204653ed11087774bf7ab23f847cf4dc900_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204653ed11087774bf7ab23f847cf4dc900_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,271 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046a9eca4fac9d46b0a38129800fefa329_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412046a9eca4fac9d46b0a38129800fefa329_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,272 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047fb35cd1cc4b44aa82b20a34fac63a73_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412047fb35cd1cc4b44aa82b20a34fac63a73_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,273 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a135276b276b4d7eb30a0b6e9d2c07a5_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a135276b276b4d7eb30a0b6e9d2c07a5_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,273 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a1e5ff21d64542d38ca6c0f9bb9a7761_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204a1e5ff21d64542d38ca6c0f9bb9a7761_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,274 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa913408bfa6431aa83f03223abf41b5_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aa913408bfa6431aa83f03223abf41b5_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aab1465346ac4ba8b431196a366b9475_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204aab1465346ac4ba8b431196a366b9475_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,276 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bde22056e13040c6b06b29b1ff3d84ce_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bde22056e13040c6b06b29b1ff3d84ce_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cd2875e24b3e46d08267b832c2da502f_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204cd2875e24b3e46d08267b832c2da502f_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddf0339302c44de098222e8f28bba3ca_896711ea717d7f14b2e304328616f183 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204ddf0339302c44de098222e8f28bba3ca_896711ea717d7f14b2e304328616f183 2024-12-04T06:42:43,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:42:43,280 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,282 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:42:43,283 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:42:43,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,284 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:42:43,284 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294563284"}]},"ts":"9223372036854775807"} 2024-12-04T06:42:43,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:42:43,286 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 896711ea717d7f14b2e304328616f183, NAME => 'TestAcidGuarantees,,1733294529235.896711ea717d7f14b2e304328616f183.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:42:43,286 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:42:43,286 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294563286"}]},"ts":"9223372036854775807"} 2024-12-04T06:42:43,288 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:42:43,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-04T06:42:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T06:42:43,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:42:43,347 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:43,356 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 399), ProcessCount=11 (was 11), AvailableMemoryMB=5896 (was 5993) 2024-12-04T06:42:43,365 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=5895 2024-12-04T06:42:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:42:43,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:42:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:42:43,368 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:42:43,368 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:43,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-04T06:42:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:43,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:42:43,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742275_1451 (size=1061) 2024-12-04T06:42:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:43,775 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:42:43,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742276_1452 (size=53) 2024-12-04T06:42:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 78d54f8244a2fd6e93eab21d996d309d, disabling compactions & flushes 2024-12-04T06:42:44,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. after waiting 0 ms 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78d54f8244a2fd6e93eab21d996d309d: Waiting for close lock at 1733294564180Disabling compacts and flushes for region at 1733294564180Disabling writes for close at 1733294564180Writing region close event to WAL at 1733294564180Closed at 1733294564180 2024-12-04T06:42:44,182 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:42:44,182 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294564182"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294564182"}]},"ts":"1733294564182"} 2024-12-04T06:42:44,183 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:42:44,184 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:42:44,184 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294564184"}]},"ts":"1733294564184"} 2024-12-04T06:42:44,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:42:44,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, ASSIGN}] 2024-12-04T06:42:44,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, ASSIGN 2024-12-04T06:42:44,186 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:42:44,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=78d54f8244a2fd6e93eab21d996d309d, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:44,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, ASSIGN because future has completed 2024-12-04T06:42:44,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:42:44,493 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,493 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:42:44,493 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,494 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:42:44,494 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,494 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,495 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,496 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:44,496 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78d54f8244a2fd6e93eab21d996d309d columnFamilyName A 2024-12-04T06:42:44,496 DEBUG [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:44,496 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(327): Store=78d54f8244a2fd6e93eab21d996d309d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:44,496 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,497 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:44,497 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78d54f8244a2fd6e93eab21d996d309d columnFamilyName B 2024-12-04T06:42:44,497 DEBUG [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:44,498 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(327): Store=78d54f8244a2fd6e93eab21d996d309d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:44,498 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,498 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:42:44,499 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78d54f8244a2fd6e93eab21d996d309d columnFamilyName C 2024-12-04T06:42:44,499 DEBUG [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:42:44,499 INFO [StoreOpener-78d54f8244a2fd6e93eab21d996d309d-1 {}] regionserver.HStore(327): Store=78d54f8244a2fd6e93eab21d996d309d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:42:44,499 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,499 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,500 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,500 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,500 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,500 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,501 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:42:44,502 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,503 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:42:44,503 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 78d54f8244a2fd6e93eab21d996d309d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66626608, jitterRate=-0.007186174392700195}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:42:44,503 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:44,504 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 78d54f8244a2fd6e93eab21d996d309d: Running coprocessor pre-open hook at 1733294564494Writing region info on filesystem at 1733294564494Initializing all the Stores at 1733294564494Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294564494Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294564495 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294564495Cleaning up temporary data from old regions at 1733294564500 (+5 ms)Running coprocessor post-open hooks at 1733294564503 (+3 ms)Region opened successfully at 1733294564504 (+1 ms) 2024-12-04T06:42:44,504 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., pid=114, masterSystemTime=1733294564490 2024-12-04T06:42:44,506 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,506 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:44,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=78d54f8244a2fd6e93eab21d996d309d, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:44,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:42:44,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-04T06:42:44,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 in 170 msec 2024-12-04T06:42:44,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-04T06:42:44,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, ASSIGN in 325 msec 2024-12-04T06:42:44,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:42:44,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294564513"}]},"ts":"1733294564513"} 2024-12-04T06:42:44,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:42:44,515 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:42:44,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-04T06:42:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T06:42:45,517 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:45,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4b685d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,519 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3f7a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2953059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,521 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,524 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:42:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:42:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0b2c31 2024-12-04T06:42:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:42:45,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:42:45,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499ef816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,527 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd75537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,528 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68efe3d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135be63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,530 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b555a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,531 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569bf8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432aef64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35779cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,534 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2081f3ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a177044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,535 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d48a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d9c9ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbbba3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4ffbbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,539 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79baa67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b052618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,540 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119d3eb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71211023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bc9769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39396490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b05e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460a8c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6877c13a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50da939f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2621535d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2348ec04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694dc39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:42:45,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:42:45,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:42:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:42:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:42:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32536b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:42:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:42:45,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:42:45,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:42:45,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c22641a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:42:45,555 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,555 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:45,556 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,557 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-04T06:42:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,557 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:45,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:45,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,564 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,568 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,569 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,574 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,575 DEBUG [Thread-2041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:45,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,582 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:42:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:42:45,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:42:45,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:42:45,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2] 2024-12-04T06:42:45,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/f3fa493a44744d94a1bb6b78fcdf15a2 is 50, key is test_row_0/A:col10/1733294565569/Put/seqid=0 2024-12-04T06:42:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742277_1453 (size=14341) 2024-12-04T06:42:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294625605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294625606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294625607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294625611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294625620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:45,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:45,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294625727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294625727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294625727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294625727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294625727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:45,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294625938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294625946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294625946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294625946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294625947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/f3fa493a44744d94a1bb6b78fcdf15a2 2024-12-04T06:42:46,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/1e34c7339e34437ea79b43b8eb9792d1 is 50, key is test_row_0/B:col10/1733294565569/Put/seqid=0 2024-12-04T06:42:46,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742278_1454 (size=12001) 2024-12-04T06:42:46,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:46,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294626256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294626266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294626266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294626267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294626267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/1e34c7339e34437ea79b43b8eb9792d1 2024-12-04T06:42:46,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d0c0d0ff2b0d42e882df21fded5710d1 is 50, key is test_row_0/C:col10/1733294565569/Put/seqid=0 2024-12-04T06:42:46,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742279_1455 (size=12001) 2024-12-04T06:42:46,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d0c0d0ff2b0d42e882df21fded5710d1 2024-12-04T06:42:46,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/f3fa493a44744d94a1bb6b78fcdf15a2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2 2024-12-04T06:42:46,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2, entries=200, sequenceid=14, filesize=14.0 K 2024-12-04T06:42:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/1e34c7339e34437ea79b43b8eb9792d1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1 2024-12-04T06:42:46,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:46,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T06:42:46,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d0c0d0ff2b0d42e882df21fded5710d1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1 2024-12-04T06:42:46,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-04T06:42:46,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 78d54f8244a2fd6e93eab21d996d309d in 911ms, sequenceid=14, compaction requested=false 2024-12-04T06:42:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:46,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:46,632 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:46,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/1877a36f24fb43679d238d0deef97ae5 is 50, key is test_row_0/A:col10/1733294565618/Put/seqid=0 2024-12-04T06:42:46,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742280_1456 (size=12001) 2024-12-04T06:42:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:46,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:46,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294626796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294626797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294626806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294626807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294626808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294626917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294626917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294626917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294626917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:46,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:46,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294626927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,048 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/1877a36f24fb43679d238d0deef97ae5 2024-12-04T06:42:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/37d3b0ecdfc9426bb90d4b355c0ea000 is 50, key is test_row_0/B:col10/1733294565618/Put/seqid=0 2024-12-04T06:42:47,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742281_1457 (size=12001) 2024-12-04T06:42:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294627136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294627137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294627137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294627137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294627147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294627446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294627447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294627447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294627457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:47,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:47,462 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/37d3b0ecdfc9426bb90d4b355c0ea000 2024-12-04T06:42:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/91a5ee9a241042b4aed0c9010ca70804 is 50, key is test_row_0/C:col10/1733294565618/Put/seqid=0 2024-12-04T06:42:47,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742282_1458 (size=12001) 2024-12-04T06:42:47,477 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/91a5ee9a241042b4aed0c9010ca70804 2024-12-04T06:42:47,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/1877a36f24fb43679d238d0deef97ae5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5 2024-12-04T06:42:47,483 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:42:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/37d3b0ecdfc9426bb90d4b355c0ea000 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000 2024-12-04T06:42:47,487 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:42:47,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/91a5ee9a241042b4aed0c9010ca70804 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804 2024-12-04T06:42:47,490 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804, entries=150, sequenceid=33, filesize=11.7 K 2024-12-04T06:42:47,491 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 78d54f8244a2fd6e93eab21d996d309d in 859ms, sequenceid=33, compaction requested=false 2024-12-04T06:42:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-04T06:42:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-04T06:42:47,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-04T06:42:47,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-12-04T06:42:47,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9380 sec 2024-12-04T06:42:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-04T06:42:47,707 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:47,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-04T06:42:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:47,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:47,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:47,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:47,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-04T06:42:47,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:47,862 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:47,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/7097b96e54044081b9bfaaf3ad71c2c8 is 50, key is test_row_0/A:col10/1733294566805/Put/seqid=0 2024-12-04T06:42:47,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742283_1459 (size=12001) 2024-12-04T06:42:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:47,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294627993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294627994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294627995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294628006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294628008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294628117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294628117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294628117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294628117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294628126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,271 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/7097b96e54044081b9bfaaf3ad71c2c8 2024-12-04T06:42:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/4228c3a51bf14ea6a237d84840eb4d0f is 50, key is test_row_0/B:col10/1733294566805/Put/seqid=0 2024-12-04T06:42:48,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742284_1460 (size=12001) 2024-12-04T06:42:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294628337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294628337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294628337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294628337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294628337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,420 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:42:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294628647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294628656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294628656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294628657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294628657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:48,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:48,685 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/4228c3a51bf14ea6a237d84840eb4d0f 2024-12-04T06:42:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2958d7c74b46493c80faa63f8432eb35 is 50, key is test_row_0/C:col10/1733294566805/Put/seqid=0 2024-12-04T06:42:48,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742285_1461 (size=12001) 2024-12-04T06:42:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:49,095 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2958d7c74b46493c80faa63f8432eb35 2024-12-04T06:42:49,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/7097b96e54044081b9bfaaf3ad71c2c8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8 2024-12-04T06:42:49,102 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T06:42:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/4228c3a51bf14ea6a237d84840eb4d0f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f 2024-12-04T06:42:49,106 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T06:42:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2958d7c74b46493c80faa63f8432eb35 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35 2024-12-04T06:42:49,109 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35, entries=150, sequenceid=47, filesize=11.7 K 2024-12-04T06:42:49,110 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 78d54f8244a2fd6e93eab21d996d309d in 1248ms, sequenceid=47, compaction requested=true 2024-12-04T06:42:49,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:49,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:49,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-04T06:42:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-04T06:42:49,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-04T06:42:49,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-12-04T06:42:49,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4060 sec 2024-12-04T06:42:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:49,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:49,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/b8caaf923efd4a70a8eedfc6b5429016 is 50, key is test_row_0/A:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:49,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742286_1462 (size=14341) 2024-12-04T06:42:49,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294629187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294629188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294629196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294629197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294629199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294629306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294629306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294629306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294629317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294629317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294629526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294629527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294629527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294629537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294629537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/b8caaf923efd4a70a8eedfc6b5429016 2024-12-04T06:42:49,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/05a7cfceb9854856a91a9b0da572d8f1 is 50, key is test_row_0/B:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:49,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742287_1463 (size=12001) 2024-12-04T06:42:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T06:42:49,857 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294629846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294629846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294629846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-04T06:42:49,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:49,859 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:49,860 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:49,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294629857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294629857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:49,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/05a7cfceb9854856a91a9b0da572d8f1 2024-12-04T06:42:49,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f985330c037f4b82bdb15aebff846533 is 50, key is test_row_0/C:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:50,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742288_1464 (size=12001) 2024-12-04T06:42:50,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T06:42:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T06:42:50,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:50,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:50,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T06:42:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294630367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294630367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294630367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294630367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294630367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:50,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f985330c037f4b82bdb15aebff846533 2024-12-04T06:42:50,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/b8caaf923efd4a70a8eedfc6b5429016 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016 2024-12-04T06:42:50,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016, entries=200, sequenceid=66, filesize=14.0 K 2024-12-04T06:42:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/05a7cfceb9854856a91a9b0da572d8f1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1 2024-12-04T06:42:50,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1, entries=150, sequenceid=66, filesize=11.7 K 2024-12-04T06:42:50,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f985330c037f4b82bdb15aebff846533 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533 2024-12-04T06:42:50,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533, entries=150, sequenceid=66, filesize=11.7 K 2024-12-04T06:42:50,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 78d54f8244a2fd6e93eab21d996d309d in 1250ms, sequenceid=66, compaction requested=true 2024-12-04T06:42:50,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:50,420 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:50,421 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:50,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:42:50,424 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,424 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,424 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=46.9 K 2024-12-04T06:42:50,424 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=51.4 K 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e34c7339e34437ea79b43b8eb9792d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733294565569 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3fa493a44744d94a1bb6b78fcdf15a2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733294565569 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d3b0ecdfc9426bb90d4b355c0ea000, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294565604 2024-12-04T06:42:50,424 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1877a36f24fb43679d238d0deef97ae5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294565604 2024-12-04T06:42:50,425 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 4228c3a51bf14ea6a237d84840eb4d0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733294566792 2024-12-04T06:42:50,425 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7097b96e54044081b9bfaaf3ad71c2c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733294566792 2024-12-04T06:42:50,425 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a7cfceb9854856a91a9b0da572d8f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:50,425 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8caaf923efd4a70a8eedfc6b5429016, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:50,433 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:50,433 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#392 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:50,433 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/9e7297cfeb534de09f49a8f832e5fbf8 is 50, key is test_row_0/B:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:50,433 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/d890c6c4e52941db8e237f3164f2acc7 is 50, key is test_row_0/A:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:50,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742289_1465 (size=12139) 2024-12-04T06:42:50,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742290_1466 (size=12139) 2024-12-04T06:42:50,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-04T06:42:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,472 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:50,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c4ad62bd04de4c8b8cf0056a530ca8df is 50, key is test_row_0/A:col10/1733294569198/Put/seqid=0 2024-12-04T06:42:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742291_1467 (size=12001) 2024-12-04T06:42:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:50,841 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/9e7297cfeb534de09f49a8f832e5fbf8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/9e7297cfeb534de09f49a8f832e5fbf8 2024-12-04T06:42:50,841 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/d890c6c4e52941db8e237f3164f2acc7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/d890c6c4e52941db8e237f3164f2acc7 2024-12-04T06:42:50,846 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into d890c6c4e52941db8e237f3164f2acc7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:50,846 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into 9e7297cfeb534de09f49a8f832e5fbf8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:50,846 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=12, startTime=1733294570420; duration=0sec 2024-12-04T06:42:50,846 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=12, startTime=1733294570421; duration=0sec 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:42:50,846 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T06:42:50,847 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T06:42:50,847 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:42:50,847 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:50,847 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=46.9 K 2024-12-04T06:42:50,848 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c0d0ff2b0d42e882df21fded5710d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733294565569 2024-12-04T06:42:50,848 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a5ee9a241042b4aed0c9010ca70804, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733294565604 2024-12-04T06:42:50,848 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2958d7c74b46493c80faa63f8432eb35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733294566792 2024-12-04T06:42:50,848 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f985330c037f4b82bdb15aebff846533, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:50,858 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#395 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:50,858 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/9590b0a24d1b460295bdaf9008a4332f is 50, key is test_row_0/C:col10/1733294569169/Put/seqid=0 2024-12-04T06:42:50,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742292_1468 (size=12139) 2024-12-04T06:42:50,881 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c4ad62bd04de4c8b8cf0056a530ca8df 2024-12-04T06:42:50,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3808dc8c57a34c39a631381697aa59ab is 50, key is test_row_0/B:col10/1733294569198/Put/seqid=0 2024-12-04T06:42:50,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742293_1469 (size=12001) 2024-12-04T06:42:50,894 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3808dc8c57a34c39a631381697aa59ab 2024-12-04T06:42:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/19c2147ba1bf474e9c2cf3235b3e94c8 is 50, key is test_row_0/C:col10/1733294569198/Put/seqid=0 2024-12-04T06:42:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742294_1470 (size=12001) 2024-12-04T06:42:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:51,271 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/9590b0a24d1b460295bdaf9008a4332f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/9590b0a24d1b460295bdaf9008a4332f 2024-12-04T06:42:51,275 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into 9590b0a24d1b460295bdaf9008a4332f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:51,275 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:51,275 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=12, startTime=1733294570421; duration=0sec 2024-12-04T06:42:51,275 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:51,276 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:42:51,305 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/19c2147ba1bf474e9c2cf3235b3e94c8 2024-12-04T06:42:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c4ad62bd04de4c8b8cf0056a530ca8df as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df 2024-12-04T06:42:51,312 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df, entries=150, sequenceid=79, filesize=11.7 K 2024-12-04T06:42:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3808dc8c57a34c39a631381697aa59ab as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab 2024-12-04T06:42:51,316 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab, entries=150, sequenceid=79, filesize=11.7 K 2024-12-04T06:42:51,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/19c2147ba1bf474e9c2cf3235b3e94c8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8 2024-12-04T06:42:51,320 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8, entries=150, sequenceid=79, filesize=11.7 K 2024-12-04T06:42:51,321 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 78d54f8244a2fd6e93eab21d996d309d in 849ms, sequenceid=79, compaction requested=false 2024-12-04T06:42:51,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:51,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:51,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-04T06:42:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-04T06:42:51,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T06:42:51,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-12-04T06:42:51,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4670 sec 2024-12-04T06:42:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:51,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:42:51,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:51,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/745f5b85e5044736a816ceeeca54249a is 50, key is test_row_0/A:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:51,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742295_1471 (size=14337) 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733294571407, max timestamp=1733294571449Num uniques 859; 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733294571407, max timestamp=1733294571449Num uniques 859; 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733294571407, max timestamp=1733294571449Num uniques 859; 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T06:42:51,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-04T06:42:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294631481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294631482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294631484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294631488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294631489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294631606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294631606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294631607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294631617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294631617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/745f5b85e5044736a816ceeeca54249a 2024-12-04T06:42:51,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/f5349ec3e65144e781a2925dcfde7f72 is 50, key is test_row_0/B:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:51,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742296_1472 (size=9657) 2024-12-04T06:42:51,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/f5349ec3e65144e781a2925dcfde7f72 2024-12-04T06:42:51,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cc3b92c0d12747dc9672121fbc616e4a is 50, key is test_row_0/C:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:51,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742297_1473 (size=9657) 2024-12-04T06:42:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294631827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294631827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294631827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294631836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294631836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T06:42:52,007 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:52,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-04T06:42:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:52,009 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:52,010 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:52,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:52,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294632147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294632147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294632147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T06:42:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:52,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294632156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294632157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cc3b92c0d12747dc9672121fbc616e4a 2024-12-04T06:42:52,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/745f5b85e5044736a816ceeeca54249a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a 2024-12-04T06:42:52,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a, entries=200, sequenceid=92, filesize=14.0 K 2024-12-04T06:42:52,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/f5349ec3e65144e781a2925dcfde7f72 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72 2024-12-04T06:42:52,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72, entries=100, sequenceid=92, filesize=9.4 K 2024-12-04T06:42:52,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cc3b92c0d12747dc9672121fbc616e4a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a 2024-12-04T06:42:52,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a, entries=100, sequenceid=92, filesize=9.4 K 2024-12-04T06:42:52,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 78d54f8244a2fd6e93eab21d996d309d in 849ms, sequenceid=92, compaction requested=true 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:52,239 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:52,239 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38477 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:42:52,240 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,240 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,240 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/d890c6c4e52941db8e237f3164f2acc7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=37.6 K 2024-12-04T06:42:52,240 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/9e7297cfeb534de09f49a8f832e5fbf8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=33.0 K 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7297cfeb534de09f49a8f832e5fbf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:52,240 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d890c6c4e52941db8e237f3164f2acc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:52,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3808dc8c57a34c39a631381697aa59ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733294569185 2024-12-04T06:42:52,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ad62bd04de4c8b8cf0056a530ca8df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733294569185 2024-12-04T06:42:52,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f5349ec3e65144e781a2925dcfde7f72, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294571388 2024-12-04T06:42:52,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745f5b85e5044736a816ceeeca54249a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294571384 2024-12-04T06:42:52,248 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:52,249 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/cb6129d2f0c84023a294b4da00d505e2 is 50, key is test_row_0/B:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:52,250 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:52,250 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/ddc71c51eef84b0dbc2fb0f5c8d39164 is 50, key is test_row_0/A:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:52,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742298_1474 (size=12241) 2024-12-04T06:42:52,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742299_1475 (size=12241) 2024-12-04T06:42:52,265 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/cb6129d2f0c84023a294b4da00d505e2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/cb6129d2f0c84023a294b4da00d505e2 2024-12-04T06:42:52,269 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into cb6129d2f0c84023a294b4da00d505e2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:52,269 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:52,269 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=13, startTime=1733294572239; duration=0sec 2024-12-04T06:42:52,269 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:52,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:42:52,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:52,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:52,270 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:42:52,270 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,271 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/9590b0a24d1b460295bdaf9008a4332f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=33.0 K 2024-12-04T06:42:52,271 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9590b0a24d1b460295bdaf9008a4332f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733294567990 2024-12-04T06:42:52,271 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c2147ba1bf474e9c2cf3235b3e94c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733294569185 2024-12-04T06:42:52,272 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3b92c0d12747dc9672121fbc616e4a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294571388 2024-12-04T06:42:52,278 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:52,278 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d35c6c4192454723a57714454ea8816f is 50, key is test_row_0/C:col10/1733294571388/Put/seqid=0 2024-12-04T06:42:52,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742300_1476 (size=12241) 2024-12-04T06:42:52,285 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d35c6c4192454723a57714454ea8816f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d35c6c4192454723a57714454ea8816f 2024-12-04T06:42:52,289 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into d35c6c4192454723a57714454ea8816f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:52,289 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:52,289 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=13, startTime=1733294572239; duration=0sec 2024-12-04T06:42:52,289 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:52,289 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:42:52,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-04T06:42:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:52,315 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-04T06:42:52,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:42:52,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/fe69528ad3214a21b17e46e0c8b9e857 is 50, key is test_row_0/A:col10/1733294571483/Put/seqid=0 2024-12-04T06:42:52,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742301_1477 (size=12001) 2024-12-04T06:42:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:52,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:52,668 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/ddc71c51eef84b0dbc2fb0f5c8d39164 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/ddc71c51eef84b0dbc2fb0f5c8d39164 2024-12-04T06:42:52,671 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into ddc71c51eef84b0dbc2fb0f5c8d39164(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:52,671 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:52,671 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=13, startTime=1733294572239; duration=0sec 2024-12-04T06:42:52,672 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:52,672 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:42:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294632678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294632686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294632688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294632690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294632690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,725 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/fe69528ad3214a21b17e46e0c8b9e857 2024-12-04T06:42:52,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/123bbccc6b6e4cc18d7fb95e193fb3f5 is 50, key is test_row_0/B:col10/1733294571483/Put/seqid=0 2024-12-04T06:42:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742302_1478 (size=12001) 2024-12-04T06:42:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294632797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294632797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294632806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294632806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294632806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294633007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294633007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294633027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294633027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294633027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,140 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/123bbccc6b6e4cc18d7fb95e193fb3f5 2024-12-04T06:42:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/41b842e16e82454da6ebba1ac0bba146 is 50, key is test_row_0/C:col10/1733294571483/Put/seqid=0 2024-12-04T06:42:53,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742303_1479 (size=12001) 2024-12-04T06:42:53,187 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/41b842e16e82454da6ebba1ac0bba146 2024-12-04T06:42:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/fe69528ad3214a21b17e46e0c8b9e857 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857 2024-12-04T06:42:53,197 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857, entries=150, sequenceid=127, filesize=11.7 K 2024-12-04T06:42:53,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/123bbccc6b6e4cc18d7fb95e193fb3f5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5 2024-12-04T06:42:53,203 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5, entries=150, sequenceid=127, filesize=11.7 K 2024-12-04T06:42:53,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/41b842e16e82454da6ebba1ac0bba146 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146 2024-12-04T06:42:53,214 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146, entries=150, sequenceid=127, filesize=11.7 K 2024-12-04T06:42:53,215 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=53.67 KB/54960 for 78d54f8244a2fd6e93eab21d996d309d in 900ms, sequenceid=127, compaction requested=false 2024-12-04T06:42:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:53,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T06:42:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T06:42:53,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T06:42:53,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-12-04T06:42:53,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2160 sec 2024-12-04T06:42:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:53,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:53,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:53,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/27e2283c75214307870e16e840a6ddc9 is 50, key is test_row_0/A:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:53,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742304_1480 (size=16931) 2024-12-04T06:42:53,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294633370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294633371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294633377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294633379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294633380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294633486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294633497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294633497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294633497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294633507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294633696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294633717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294633717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294633717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294633726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:53,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/27e2283c75214307870e16e840a6ddc9 2024-12-04T06:42:53,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/210aad7ff33b4ee1a892ff6d7bdafc6c is 50, key is test_row_0/B:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:53,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742305_1481 (size=12151) 2024-12-04T06:42:54,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294634016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294634037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294634037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294634037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294634037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T06:42:54,157 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:54,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-04T06:42:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:54,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:54,161 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:54,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:54,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/210aad7ff33b4ee1a892ff6d7bdafc6c 2024-12-04T06:42:54,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2f24ff7cb1274184b4ceeffc5b04add1 is 50, key is test_row_0/C:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:54,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742306_1482 (size=12151) 2024-12-04T06:42:54,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2f24ff7cb1274184b4ceeffc5b04add1 2024-12-04T06:42:54,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/27e2283c75214307870e16e840a6ddc9 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9 2024-12-04T06:42:54,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9, entries=250, sequenceid=141, filesize=16.5 K 2024-12-04T06:42:54,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/210aad7ff33b4ee1a892ff6d7bdafc6c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c 2024-12-04T06:42:54,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T06:42:54,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/2f24ff7cb1274184b4ceeffc5b04add1 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1 2024-12-04T06:42:54,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1, entries=150, sequenceid=141, filesize=11.9 K 2024-12-04T06:42:54,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 78d54f8244a2fd6e93eab21d996d309d in 868ms, sequenceid=141, compaction requested=true 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:54,206 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:54,206 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:54,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:54,207 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:54,207 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:42:54,207 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:54,207 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/ddc71c51eef84b0dbc2fb0f5c8d39164, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=40.2 K 2024-12-04T06:42:54,207 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc71c51eef84b0dbc2fb0f5c8d39164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294569185 2024-12-04T06:42:54,208 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:54,208 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:42:54,208 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:54,208 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/cb6129d2f0c84023a294b4da00d505e2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=35.5 K 2024-12-04T06:42:54,208 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe69528ad3214a21b17e46e0c8b9e857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294571483 2024-12-04T06:42:54,208 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6129d2f0c84023a294b4da00d505e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294569185 2024-12-04T06:42:54,208 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e2283c75214307870e16e840a6ddc9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:54,209 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 123bbccc6b6e4cc18d7fb95e193fb3f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294571483 2024-12-04T06:42:54,209 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 210aad7ff33b4ee1a892ff6d7bdafc6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:54,223 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:54,223 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/dd101d547e694181bc0fca7c00ec7335 is 50, key is test_row_0/B:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:54,234 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:54,235 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/9eb23bd6b83b498e9a342a43c10c27b0 is 50, key is test_row_0/A:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:54,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742307_1483 (size=12493) 2024-12-04T06:42:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742308_1484 (size=12493) 2024-12-04T06:42:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:54,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-04T06:42:54,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:54,314 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a2c550bd7e144b549b83067d335ff4b3 is 50, key is test_row_0/A:col10/1733294573379/Put/seqid=0 2024-12-04T06:42:54,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742309_1485 (size=12151) 2024-12-04T06:42:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:54,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294634590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294634590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294634591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294634592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294634592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,657 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/dd101d547e694181bc0fca7c00ec7335 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dd101d547e694181bc0fca7c00ec7335 2024-12-04T06:42:54,657 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/9eb23bd6b83b498e9a342a43c10c27b0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9eb23bd6b83b498e9a342a43c10c27b0 2024-12-04T06:42:54,664 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into 9eb23bd6b83b498e9a342a43c10c27b0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:54,664 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into dd101d547e694181bc0fca7c00ec7335(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:54,664 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=13, startTime=1733294574206; duration=0sec 2024-12-04T06:42:54,664 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=13, startTime=1733294574206; duration=0sec 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:42:54,664 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:54,665 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:54,665 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:42:54,665 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:54,665 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d35c6c4192454723a57714454ea8816f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=35.5 K 2024-12-04T06:42:54,665 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35c6c4192454723a57714454ea8816f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733294569185 2024-12-04T06:42:54,665 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b842e16e82454da6ebba1ac0bba146, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733294571483 2024-12-04T06:42:54,666 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f24ff7cb1274184b4ceeffc5b04add1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:54,673 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:54,674 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/e27604925cb9407da6373b8fb3f63a4b is 50, key is test_row_0/C:col10/1733294572664/Put/seqid=0 2024-12-04T06:42:54,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742310_1486 (size=12493) 2024-12-04T06:42:54,689 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/e27604925cb9407da6373b8fb3f63a4b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/e27604925cb9407da6373b8fb3f63a4b 2024-12-04T06:42:54,696 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into e27604925cb9407da6373b8fb3f63a4b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:54,696 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:54,696 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=13, startTime=1733294574206; duration=0sec 2024-12-04T06:42:54,696 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:54,696 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:42:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294634716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294634716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294634716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294634716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294634717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,739 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a2c550bd7e144b549b83067d335ff4b3 2024-12-04T06:42:54,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3e4d217f94dd4e34970add638c5968ae is 50, key is test_row_0/B:col10/1733294573379/Put/seqid=0 2024-12-04T06:42:54,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742311_1487 (size=12151) 2024-12-04T06:42:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:54,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294634937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294634937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294634937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294634937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294634937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,156 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3e4d217f94dd4e34970add638c5968ae 2024-12-04T06:42:55,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/53c3b33c83dc4a41a3f7a41562878c7b is 50, key is test_row_0/C:col10/1733294573379/Put/seqid=0 2024-12-04T06:42:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742312_1488 (size=12151) 2024-12-04T06:42:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294635257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294635257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294635257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294635257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294635257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:55,568 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/53c3b33c83dc4a41a3f7a41562878c7b 2024-12-04T06:42:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a2c550bd7e144b549b83067d335ff4b3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3 2024-12-04T06:42:55,578 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3, entries=150, sequenceid=160, filesize=11.9 K 2024-12-04T06:42:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3e4d217f94dd4e34970add638c5968ae as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae 2024-12-04T06:42:55,583 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae, entries=150, sequenceid=160, filesize=11.9 K 2024-12-04T06:42:55,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/53c3b33c83dc4a41a3f7a41562878c7b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b 2024-12-04T06:42:55,588 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b, entries=150, sequenceid=160, filesize=11.9 K 2024-12-04T06:42:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-04T06:42:55,590 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for 78d54f8244a2fd6e93eab21d996d309d in 1275ms, sequenceid=160, compaction requested=false 2024-12-04T06:42:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-04T06:42:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-04T06:42:55,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-04T06:42:55,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-12-04T06:42:55,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4390 sec 2024-12-04T06:42:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:55,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:55,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/3a354b3aac0246928fd52135e6c51391 is 50, key is test_row_0/A:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:55,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742313_1489 (size=12151) 2024-12-04T06:42:55,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/3a354b3aac0246928fd52135e6c51391 2024-12-04T06:42:55,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294635822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294635823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294635825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294635834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/dc991bf5b76e4f03a3c3b0d579803081 is 50, key is test_row_0/B:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:55,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294635834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742314_1490 (size=12151) 2024-12-04T06:42:55,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/dc991bf5b76e4f03a3c3b0d579803081 2024-12-04T06:42:55,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/872ecdb58eeb47fb8f9f0432b39b9a26 is 50, key is test_row_0/C:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:55,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742315_1491 (size=12151) 2024-12-04T06:42:55,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/872ecdb58eeb47fb8f9f0432b39b9a26 2024-12-04T06:42:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/3a354b3aac0246928fd52135e6c51391 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391 2024-12-04T06:42:55,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391, entries=150, sequenceid=179, filesize=11.9 K 2024-12-04T06:42:55,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/dc991bf5b76e4f03a3c3b0d579803081 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081 2024-12-04T06:42:55,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081, entries=150, sequenceid=179, filesize=11.9 K 2024-12-04T06:42:55,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/872ecdb58eeb47fb8f9f0432b39b9a26 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26 2024-12-04T06:42:55,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26, entries=150, sequenceid=179, filesize=11.9 K 2024-12-04T06:42:55,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 78d54f8244a2fd6e93eab21d996d309d in 135ms, sequenceid=179, compaction requested=true 2024-12-04T06:42:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:55,919 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:42:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:55,919 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:42:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:55,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:42:55,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:55,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:55,923 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:42:55,923 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:55,923 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dd101d547e694181bc0fca7c00ec7335, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=35.9 K 2024-12-04T06:42:55,924 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting dd101d547e694181bc0fca7c00ec7335, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:55,924 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4d217f94dd4e34970add638c5968ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733294573375 2024-12-04T06:42:55,924 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting dc991bf5b76e4f03a3c3b0d579803081, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:42:55,925 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:55,925 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:42:55,925 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:55,925 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9eb23bd6b83b498e9a342a43c10c27b0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=35.9 K 2024-12-04T06:42:55,926 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb23bd6b83b498e9a342a43c10c27b0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:55,929 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c550bd7e144b549b83067d335ff4b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733294573375 2024-12-04T06:42:55,929 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a354b3aac0246928fd52135e6c51391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:42:55,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:55,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:55,952 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#420 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:55,952 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/2b08f83d121e4b2aa28888a8a8e484ed is 50, key is test_row_0/A:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:55,956 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#419 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:55,957 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3cf0ca737b4c484ca6aeebd32d1c6f0e is 50, key is test_row_0/B:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:55,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e01dad55793d4a56905014e5989c65f4 is 50, key is test_row_0/A:col10/1733294575832/Put/seqid=0 2024-12-04T06:42:55,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742316_1492 (size=12595) 2024-12-04T06:42:55,983 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/2b08f83d121e4b2aa28888a8a8e484ed as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/2b08f83d121e4b2aa28888a8a8e484ed 2024-12-04T06:42:55,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742317_1493 (size=12595) 2024-12-04T06:42:55,992 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into 2b08f83d121e4b2aa28888a8a8e484ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:55,992 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:55,992 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=13, startTime=1733294575919; duration=0sec 2024-12-04T06:42:55,992 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:42:55,992 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:42:55,992 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:42:55,994 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:42:55,994 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:42:55,994 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:55,994 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/e27604925cb9407da6373b8fb3f63a4b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=35.9 K 2024-12-04T06:42:55,995 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27604925cb9407da6373b8fb3f63a4b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733294572664 2024-12-04T06:42:55,995 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c3b33c83dc4a41a3f7a41562878c7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733294573375 2024-12-04T06:42:55,996 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872ecdb58eeb47fb8f9f0432b39b9a26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:42:55,996 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3cf0ca737b4c484ca6aeebd32d1c6f0e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3cf0ca737b4c484ca6aeebd32d1c6f0e 2024-12-04T06:42:55,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294635978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294635979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294635980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294635985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294635997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,004 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into 3cf0ca737b4c484ca6aeebd32d1c6f0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:56,004 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:56,004 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=13, startTime=1733294575919; duration=0sec 2024-12-04T06:42:56,004 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:56,004 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:42:56,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742318_1494 (size=14541) 2024-12-04T06:42:56,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e01dad55793d4a56905014e5989c65f4 2024-12-04T06:42:56,012 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:42:56,013 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/dcac359e1b184a09aa0fe6cc30601677 is 50, key is test_row_0/C:col10/1733294574589/Put/seqid=0 2024-12-04T06:42:56,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/b833e25202544fb0bf307afa45cf901b is 50, key is test_row_0/B:col10/1733294575832/Put/seqid=0 2024-12-04T06:42:56,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742319_1495 (size=12595) 2024-12-04T06:42:56,047 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/dcac359e1b184a09aa0fe6cc30601677 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/dcac359e1b184a09aa0fe6cc30601677 2024-12-04T06:42:56,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742320_1496 (size=12151) 2024-12-04T06:42:56,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/b833e25202544fb0bf307afa45cf901b 2024-12-04T06:42:56,055 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into dcac359e1b184a09aa0fe6cc30601677(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:42:56,056 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:56,056 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=13, startTime=1733294575919; duration=0sec 2024-12-04T06:42:56,056 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:42:56,056 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:42:56,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/33b32b2f756846a994adea41f49adfd6 is 50, key is test_row_0/C:col10/1733294575832/Put/seqid=0 2024-12-04T06:42:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742321_1497 (size=12151) 2024-12-04T06:42:56,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294636107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294636107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294636107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294636107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294636107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T06:42:56,317 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:56,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-04T06:42:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:56,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:56,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:56,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294636316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294636317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294636317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294636317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294636317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:56,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/33b32b2f756846a994adea41f49adfd6 2024-12-04T06:42:56,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e01dad55793d4a56905014e5989c65f4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4 2024-12-04T06:42:56,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4, entries=200, sequenceid=196, filesize=14.2 K 2024-12-04T06:42:56,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T06:42:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:56,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:56,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/b833e25202544fb0bf307afa45cf901b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b 2024-12-04T06:42:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:56,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b, entries=150, sequenceid=196, filesize=11.9 K 2024-12-04T06:42:56,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/33b32b2f756846a994adea41f49adfd6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6 2024-12-04T06:42:56,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6, entries=150, sequenceid=196, filesize=11.9 K 2024-12-04T06:42:56,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 78d54f8244a2fd6e93eab21d996d309d in 529ms, sequenceid=196, compaction requested=false 2024-12-04T06:42:56,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:56,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-04T06:42:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:56,625 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:42:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:56,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/8114117b428e4fb7b6d506b3ad63bcd8 is 50, key is test_row_0/A:col10/1733294575978/Put/seqid=0 2024-12-04T06:42:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:56,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:56,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742322_1498 (size=12151) 2024-12-04T06:42:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:56,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294636658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294636659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294636661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294636661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294636662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294636766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294636776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294636777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294636777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294636787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294636987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294637006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/8114117b428e4fb7b6d506b3ad63bcd8 2024-12-04T06:42:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/167fe31a198444469bd89d9edad4a9bb is 50, key is test_row_0/B:col10/1733294575978/Put/seqid=0 2024-12-04T06:42:57,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742323_1499 (size=12151) 2024-12-04T06:42:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294637297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294637297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294637297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294637307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294637326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,446 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/167fe31a198444469bd89d9edad4a9bb 2024-12-04T06:42:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/140611ba7e144f4eb926696d5b491bcc is 50, key is test_row_0/C:col10/1733294575978/Put/seqid=0 2024-12-04T06:42:57,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742324_1500 (size=12151) 2024-12-04T06:42:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:57,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294637816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294637816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294637816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294637827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294637847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:57,857 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/140611ba7e144f4eb926696d5b491bcc 2024-12-04T06:42:57,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/8114117b428e4fb7b6d506b3ad63bcd8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8 2024-12-04T06:42:57,864 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8, entries=150, sequenceid=214, filesize=11.9 K 2024-12-04T06:42:57,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/167fe31a198444469bd89d9edad4a9bb as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb 2024-12-04T06:42:57,868 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb, entries=150, sequenceid=214, filesize=11.9 K 2024-12-04T06:42:57,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/140611ba7e144f4eb926696d5b491bcc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc 2024-12-04T06:42:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc, entries=150, sequenceid=214, filesize=11.9 K 2024-12-04T06:42:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 78d54f8244a2fd6e93eab21d996d309d in 1247ms, sequenceid=214, compaction requested=true 2024-12-04T06:42:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T06:42:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T06:42:57,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T06:42:57,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5540 sec 2024-12-04T06:42:57,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5590 sec 2024-12-04T06:42:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T06:42:58,467 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:58,467 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-04T06:42:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T06:42:58,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:58,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:58,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T06:42:58,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:58,622 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:58,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e86ed0b8c74c4847ae3a99fa02110b8a is 50, key is test_row_0/A:col10/1733294576661/Put/seqid=0 2024-12-04T06:42:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742325_1501 (size=12151) 2024-12-04T06:42:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T06:42:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:58,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:58,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294638864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294638867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294638870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:58,880 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in 78d54f8244a2fd6e93eab21d996d309d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:58,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294638872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294638873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294638987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294638987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294638996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294638996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,031 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e86ed0b8c74c4847ae3a99fa02110b8a 2024-12-04T06:42:59,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/5b4a8143f3cb4aa282cb0c5d78b33355 is 50, key is test_row_0/B:col10/1733294576661/Put/seqid=0 2024-12-04T06:42:59,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742326_1502 (size=12151) 2024-12-04T06:42:59,081 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:42:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T06:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294639207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294639207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294639216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294639216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,443 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/5b4a8143f3cb4aa282cb0c5d78b33355 2024-12-04T06:42:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f77780af848e4c6c9515524f22a11af0 is 50, key is test_row_0/C:col10/1733294576661/Put/seqid=0 2024-12-04T06:42:59,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742327_1503 (size=12151) 2024-12-04T06:42:59,453 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f77780af848e4c6c9515524f22a11af0 2024-12-04T06:42:59,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/e86ed0b8c74c4847ae3a99fa02110b8a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a 2024-12-04T06:42:59,459 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a, entries=150, sequenceid=231, filesize=11.9 K 2024-12-04T06:42:59,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/5b4a8143f3cb4aa282cb0c5d78b33355 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355 2024-12-04T06:42:59,463 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355, entries=150, sequenceid=231, filesize=11.9 K 2024-12-04T06:42:59,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f77780af848e4c6c9515524f22a11af0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0 2024-12-04T06:42:59,466 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0, entries=150, sequenceid=231, filesize=11.9 K 2024-12-04T06:42:59,467 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 78d54f8244a2fd6e93eab21d996d309d in 845ms, sequenceid=231, compaction requested=true 2024-12-04T06:42:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:42:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:59,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-04T06:42:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-04T06:42:59,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-04T06:42:59,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-12-04T06:42:59,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0040 sec 2024-12-04T06:42:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:42:59,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:42:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:42:59,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a7a1770f325a4f96b4c86654abc476b6 is 50, key is test_row_0/A:col10/1733294578872/Put/seqid=0 2024-12-04T06:42:59,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742328_1504 (size=14541) 2024-12-04T06:42:59,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294639562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294639563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294639565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294639573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T06:42:59,607 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:42:59,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:42:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-04T06:42:59,609 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:42:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:42:59,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:42:59,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:42:59,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294639676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294639687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294639687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294639687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:42:59,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:42:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:59,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294639897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:42:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:42:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:42:59,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:42:59,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294639907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294639907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:42:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294639907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:42:59,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:42:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:42:59,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a7a1770f325a4f96b4c86654abc476b6 2024-12-04T06:42:59,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2bbf941afe9843cb89879707b585b5d2 is 50, key is test_row_0/B:col10/1733294578872/Put/seqid=0 2024-12-04T06:42:59,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742329_1505 (size=12151) 2024-12-04T06:43:00,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:43:00,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:43:00,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294640217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:43:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:43:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294640226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294640226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294640226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:43:00,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2bbf941afe9843cb89879707b585b5d2 2024-12-04T06:43:00,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/5e891eb1b5274b80955d5fea78df48de is 50, key is test_row_0/C:col10/1733294578872/Put/seqid=0 2024-12-04T06:43:00,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742330_1506 (size=12151) 2024-12-04T06:43:00,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/5e891eb1b5274b80955d5fea78df48de 2024-12-04T06:43:00,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a7a1770f325a4f96b4c86654abc476b6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6 2024-12-04T06:43:00,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6, entries=200, sequenceid=247, filesize=14.2 K 2024-12-04T06:43:00,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2bbf941afe9843cb89879707b585b5d2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2 2024-12-04T06:43:00,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2, entries=150, sequenceid=247, filesize=11.9 K 2024-12-04T06:43:00,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/5e891eb1b5274b80955d5fea78df48de as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de 2024-12-04T06:43:00,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:43:00,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de, entries=150, sequenceid=247, filesize=11.9 K 2024-12-04T06:43:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:43:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 78d54f8244a2fd6e93eab21d996d309d in 846ms, sequenceid=247, compaction requested=true 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:00,376 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T06:43:00,376 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T06:43:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:00,377 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T06:43:00,377 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 65979 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T06:43:00,377 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:43:00,377 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:43:00,377 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,378 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,378 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3cf0ca737b4c484ca6aeebd32d1c6f0e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=59.8 K 2024-12-04T06:43:00,378 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/2b08f83d121e4b2aa28888a8a8e484ed, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=64.4 K 2024-12-04T06:43:00,378 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf0ca737b4c484ca6aeebd32d1c6f0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:43:00,378 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b08f83d121e4b2aa28888a8a8e484ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01dad55793d4a56905014e5989c65f4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733294575822 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b833e25202544fb0bf307afa45cf901b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733294575831 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8114117b428e4fb7b6d506b3ad63bcd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733294575978 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 167fe31a198444469bd89d9edad4a9bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733294575978 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting e86ed0b8c74c4847ae3a99fa02110b8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733294576645 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b4a8143f3cb4aa282cb0c5d78b33355, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733294576645 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7a1770f325a4f96b4c86654abc476b6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578856 2024-12-04T06:43:00,380 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbf941afe9843cb89879707b585b5d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578870 2024-12-04T06:43:00,391 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:00,391 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3437dda9a32547428b02234b5cceb17b is 50, key is test_row_0/B:col10/1733294578872/Put/seqid=0 2024-12-04T06:43:00,393 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#434 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:00,393 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/f61bd12a1dbf4892ac4aad87459a765f is 50, key is test_row_0/A:col10/1733294578872/Put/seqid=0 2024-12-04T06:43:00,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742331_1507 (size=12765) 2024-12-04T06:43:00,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742332_1508 (size=12765) 2024-12-04T06:43:00,405 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/f61bd12a1dbf4892ac4aad87459a765f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f61bd12a1dbf4892ac4aad87459a765f 2024-12-04T06:43:00,409 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into f61bd12a1dbf4892ac4aad87459a765f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:00,409 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:00,409 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=11, startTime=1733294580376; duration=0sec 2024-12-04T06:43:00,410 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:00,410 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:43:00,410 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-04T06:43:00,411 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-04T06:43:00,411 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:43:00,411 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,411 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/dcac359e1b184a09aa0fe6cc30601677, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=59.8 K 2024-12-04T06:43:00,411 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcac359e1b184a09aa0fe6cc30601677, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733294574589 2024-12-04T06:43:00,411 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b32b2f756846a994adea41f49adfd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733294575831 2024-12-04T06:43:00,412 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140611ba7e144f4eb926696d5b491bcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733294575978 2024-12-04T06:43:00,412 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f77780af848e4c6c9515524f22a11af0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733294576645 2024-12-04T06:43:00,412 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e891eb1b5274b80955d5fea78df48de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578870 2024-12-04T06:43:00,423 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:00,423 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/5b52a0d8b5ba4870875e1bc785cfefd4 is 50, key is test_row_0/C:col10/1733294578872/Put/seqid=0 2024-12-04T06:43:00,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742333_1509 (size=12765) 2024-12-04T06:43:00,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:00,529 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:00,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:00,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/24e8315dfb6f4657bbe2edc79a674f54 is 50, key is test_row_0/A:col10/1733294579561/Put/seqid=0 2024-12-04T06:43:00,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742334_1510 (size=12301) 2024-12-04T06:43:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:00,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:43:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:43:00,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294640772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294640773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294640774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294640775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,800 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3437dda9a32547428b02234b5cceb17b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3437dda9a32547428b02234b5cceb17b 2024-12-04T06:43:00,804 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into 3437dda9a32547428b02234b5cceb17b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:00,804 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:00,804 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=11, startTime=1733294580376; duration=0sec 2024-12-04T06:43:00,804 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:00,804 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:43:00,832 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/5b52a0d8b5ba4870875e1bc785cfefd4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5b52a0d8b5ba4870875e1bc785cfefd4 2024-12-04T06:43:00,836 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into 5b52a0d8b5ba4870875e1bc785cfefd4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:00,836 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:00,836 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=11, startTime=1733294580376; duration=0sec 2024-12-04T06:43:00,836 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:00,836 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:43:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294640887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294640896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294640896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294640896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,907 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in 78d54f8244a2fd6e93eab21d996d309d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294640896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:00,940 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/24e8315dfb6f4657bbe2edc79a674f54 2024-12-04T06:43:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/43a0a038ceb145f6b11dfba5353f2cd3 is 50, key is test_row_0/B:col10/1733294579561/Put/seqid=0 2024-12-04T06:43:00,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742335_1511 (size=12301) 2024-12-04T06:43:01,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294641097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294641116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294641116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294641116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,360 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/43a0a038ceb145f6b11dfba5353f2cd3 2024-12-04T06:43:01,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/abbe2d40f84340f894a80d831d9aae4d is 50, key is test_row_0/C:col10/1733294579561/Put/seqid=0 2024-12-04T06:43:01,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742336_1512 (size=12301) 2024-12-04T06:43:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294641417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294641437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294641437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294641437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:43:01,779 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/abbe2d40f84340f894a80d831d9aae4d 2024-12-04T06:43:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/24e8315dfb6f4657bbe2edc79a674f54 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54 2024-12-04T06:43:01,789 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54, entries=150, sequenceid=264, filesize=12.0 K 2024-12-04T06:43:01,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/43a0a038ceb145f6b11dfba5353f2cd3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3 2024-12-04T06:43:01,793 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3, entries=150, sequenceid=264, filesize=12.0 K 2024-12-04T06:43:01,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/abbe2d40f84340f894a80d831d9aae4d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d 2024-12-04T06:43:01,797 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d, entries=150, sequenceid=264, filesize=12.0 K 2024-12-04T06:43:01,798 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 78d54f8244a2fd6e93eab21d996d309d in 1269ms, sequenceid=264, compaction requested=false 2024-12-04T06:43:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-04T06:43:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-04T06:43:01,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-04T06:43:01,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1890 sec 2024-12-04T06:43:01,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1940 sec 2024-12-04T06:43:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:01,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:01,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/9c4adb41517c4cdcaf802dd390cffa72 is 50, key is test_row_0/A:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742337_1513 (size=14741) 2024-12-04T06:43:01,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294641979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294641980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294641980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294641983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294642097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294642097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294642097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294642106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294642307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294642317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294642317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294642326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/9c4adb41517c4cdcaf802dd390cffa72 2024-12-04T06:43:02,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/ab91ed04f7b64046acb957a277517776 is 50, key is test_row_0/B:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:02,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742338_1514 (size=12301) 2024-12-04T06:43:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294642626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294642627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294642627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294642647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:02,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:02,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/ab91ed04f7b64046acb957a277517776 2024-12-04T06:43:02,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f1b10a9183444741b9a6e437706eabbd is 50, key is test_row_0/C:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:02,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742339_1515 (size=12301) 2024-12-04T06:43:03,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294643137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294643137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294643146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:03,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294643166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:03,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:03,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f1b10a9183444741b9a6e437706eabbd 2024-12-04T06:43:03,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/9c4adb41517c4cdcaf802dd390cffa72 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72 2024-12-04T06:43:03,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72, entries=200, sequenceid=283, filesize=14.4 K 2024-12-04T06:43:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/ab91ed04f7b64046acb957a277517776 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776 2024-12-04T06:43:03,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776, entries=150, sequenceid=283, filesize=12.0 K 2024-12-04T06:43:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/f1b10a9183444741b9a6e437706eabbd as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd 2024-12-04T06:43:03,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd, entries=150, sequenceid=283, filesize=12.0 K 2024-12-04T06:43:03,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 78d54f8244a2fd6e93eab21d996d309d in 1264ms, sequenceid=283, compaction requested=true 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:03,204 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:03,204 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:03,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:03,204 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:03,205 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:43:03,205 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:03,205 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f61bd12a1dbf4892ac4aad87459a765f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=38.9 K 2024-12-04T06:43:03,205 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61bd12a1dbf4892ac4aad87459a765f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578870 2024-12-04T06:43:03,205 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:03,205 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:43:03,205 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:03,205 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3437dda9a32547428b02234b5cceb17b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=36.5 K 2024-12-04T06:43:03,206 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e8315dfb6f4657bbe2edc79a674f54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733294579561 2024-12-04T06:43:03,206 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3437dda9a32547428b02234b5cceb17b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578870 2024-12-04T06:43:03,206 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4adb41517c4cdcaf802dd390cffa72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:03,206 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a0a038ceb145f6b11dfba5353f2cd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733294579561 2024-12-04T06:43:03,207 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ab91ed04f7b64046acb957a277517776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:03,215 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:03,215 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/8d2a5cad483c416890e12d830f070128 is 50, key is test_row_0/A:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:03,219 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:03,219 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/0c1c8455a9654738b0a5a01995354b21 is 50, key is test_row_0/B:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:03,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742340_1516 (size=13017) 2024-12-04T06:43:03,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742341_1517 (size=13017) 2024-12-04T06:43:03,241 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/8d2a5cad483c416890e12d830f070128 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8d2a5cad483c416890e12d830f070128 2024-12-04T06:43:03,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/0c1c8455a9654738b0a5a01995354b21 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/0c1c8455a9654738b0a5a01995354b21 2024-12-04T06:43:03,247 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into 8d2a5cad483c416890e12d830f070128(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:03,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:03,247 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=13, startTime=1733294583204; duration=0sec 2024-12-04T06:43:03,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:03,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:43:03,247 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:03,248 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:03,249 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:43:03,249 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:03,249 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5b52a0d8b5ba4870875e1bc785cfefd4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=36.5 K 2024-12-04T06:43:03,249 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b52a0d8b5ba4870875e1bc785cfefd4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733294578870 2024-12-04T06:43:03,249 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting abbe2d40f84340f894a80d831d9aae4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733294579561 2024-12-04T06:43:03,250 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b10a9183444741b9a6e437706eabbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:03,250 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into 0c1c8455a9654738b0a5a01995354b21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:03,250 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:03,250 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=13, startTime=1733294583204; duration=0sec 2024-12-04T06:43:03,250 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:03,250 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:43:03,258 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:03,259 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/b1fdbc0859da4488965adaf2ac29dc29 is 50, key is test_row_0/C:col10/1733294581938/Put/seqid=0 2024-12-04T06:43:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742342_1518 (size=13017) 2024-12-04T06:43:03,670 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/b1fdbc0859da4488965adaf2ac29dc29 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b1fdbc0859da4488965adaf2ac29dc29 2024-12-04T06:43:03,686 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into b1fdbc0859da4488965adaf2ac29dc29(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:03,687 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:03,687 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=13, startTime=1733294583204; duration=0sec 2024-12-04T06:43:03,687 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:03,687 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:43:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-04T06:43:03,777 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:03,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-04T06:43:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:03,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:03,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:03,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:03,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-04T06:43:03,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:03,934 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/988ce54ccced43f6b3a3bfcaee28e62c is 50, key is test_row_0/A:col10/1733294581981/Put/seqid=0 2024-12-04T06:43:03,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742343_1519 (size=12301) 2024-12-04T06:43:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:04,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. as already flushing 2024-12-04T06:43:04,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294644190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294644191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294644195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51392 deadline: 1733294644197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,209 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 78d54f8244a2fd6e93eab21d996d309d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294644307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294644307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294644316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,344 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/988ce54ccced43f6b3a3bfcaee28e62c 2024-12-04T06:43:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2c01b14a092d4e478a0886c16e1f2424 is 50, key is test_row_0/B:col10/1733294581981/Put/seqid=0 2024-12-04T06:43:04,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742344_1520 (size=12301) 2024-12-04T06:43:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294644517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294644517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294644526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,755 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2c01b14a092d4e478a0886c16e1f2424 2024-12-04T06:43:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7f675692c2784f20869d365c4b7bc7a4 is 50, key is test_row_0/C:col10/1733294581981/Put/seqid=0 2024-12-04T06:43:04,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742345_1521 (size=12301) 2024-12-04T06:43:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294644837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294644837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294644837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:04,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51386 deadline: 1733294644917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:04,927 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in 78d54f8244a2fd6e93eab21d996d309d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,164 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7f675692c2784f20869d365c4b7bc7a4 2024-12-04T06:43:05,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/988ce54ccced43f6b3a3bfcaee28e62c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c 2024-12-04T06:43:05,171 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:43:05,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2c01b14a092d4e478a0886c16e1f2424 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424 2024-12-04T06:43:05,175 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:43:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7f675692c2784f20869d365c4b7bc7a4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4 2024-12-04T06:43:05,179 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4, entries=150, sequenceid=301, filesize=12.0 K 2024-12-04T06:43:05,179 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 78d54f8244a2fd6e93eab21d996d309d in 1246ms, sequenceid=301, compaction requested=false 2024-12-04T06:43:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-04T06:43:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-04T06:43:05,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T06:43:05,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-12-04T06:43:05,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4050 sec 2024-12-04T06:43:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:05,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:05,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:05,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c572dbcc05b745b2992a4269f6fc5cb3 is 50, key is test_row_0/A:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:05,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742346_1522 (size=14741) 2024-12-04T06:43:05,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c572dbcc05b745b2992a4269f6fc5cb3 2024-12-04T06:43:05,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3ae3afa66dc04aa8a37c302f25d0fc5b is 50, key is test_row_0/B:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742347_1523 (size=12301) 2024-12-04T06:43:05,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3ae3afa66dc04aa8a37c302f25d0fc5b 2024-12-04T06:43:05,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7d999c3ee9eb49aa8979ddeb24b8f9ca is 50, key is test_row_0/C:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742348_1524 (size=12301) 2024-12-04T06:43:05,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294645378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294645388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294645389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294645506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294645506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294645507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,581 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-12-04T06:43:05,581 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:05,581 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,582 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:05,582 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-12-04T06:43:05,583 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:05,583 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,583 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:05,584 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-12-04T06:43:05,584 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:05,584 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,584 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:05,593 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-12-04T06:43:05,593 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:05,593 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,593 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:05,594 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-12-04T06:43:05,594 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:05,594 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,594 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:05,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1733294645716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51358 deadline: 1733294645716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51396 deadline: 1733294645716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 , the old value is region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., hostname=607fd5c6574c,38257,1733294430862, seqNum=2 because the exception is null or not the one we care about 2024-12-04T06:43:05,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7d999c3ee9eb49aa8979ddeb24b8f9ca 2024-12-04T06:43:05,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/c572dbcc05b745b2992a4269f6fc5cb3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3 2024-12-04T06:43:05,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3, entries=200, sequenceid=319, filesize=14.4 K 2024-12-04T06:43:05,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/3ae3afa66dc04aa8a37c302f25d0fc5b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b 2024-12-04T06:43:05,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b, entries=150, sequenceid=319, filesize=12.0 K 2024-12-04T06:43:05,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/7d999c3ee9eb49aa8979ddeb24b8f9ca as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca 2024-12-04T06:43:05,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca, entries=150, sequenceid=319, filesize=12.0 K 2024-12-04T06:43:05,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 78d54f8244a2fd6e93eab21d996d309d in 455ms, sequenceid=319, compaction requested=true 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:05,807 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78d54f8244a2fd6e93eab21d996d309d:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:05,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:05,807 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/A is initiating minor compaction (all files) 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/B is initiating minor compaction (all files) 2024-12-04T06:43:05,808 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/A in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:05,808 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/B in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:05,808 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8d2a5cad483c416890e12d830f070128, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=39.1 K 2024-12-04T06:43:05,808 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/0c1c8455a9654738b0a5a01995354b21, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=36.7 K 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1c8455a9654738b0a5a01995354b21, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2a5cad483c416890e12d830f070128, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c01b14a092d4e478a0886c16e1f2424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294581977 2024-12-04T06:43:05,808 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988ce54ccced43f6b3a3bfcaee28e62c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294581977 2024-12-04T06:43:05,809 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c572dbcc05b745b2992a4269f6fc5cb3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733294584175 2024-12-04T06:43:05,809 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ae3afa66dc04aa8a37c302f25d0fc5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733294584175 2024-12-04T06:43:05,814 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:05,814 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#A#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:05,814 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/6bed4931fa404f6983daa85b4c4ba2e2 is 50, key is test_row_0/B:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:05,814 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/69fbd7af7d4b4198be033ba6f2207614 is 50, key is test_row_0/A:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:05,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742350_1526 (size=13119) 2024-12-04T06:43:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742349_1525 (size=13119) 2024-12-04T06:43:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T06:43:05,927 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:06,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-04T06:43:06,029 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-12-04T06:43:06,029 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-12-04T06:43:06,029 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:06,029 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:06,029 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:06,029 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:06,029 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:06,029 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:06,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:06,029 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:06,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:06,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:06,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/340fc688d46b49c282efe08cfa39c7a4 is 50, key is test_row_0/A:col10/1733294586027/Put/seqid=0 2024-12-04T06:43:06,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742351_1527 (size=12301) 2024-12-04T06:43:06,221 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/6bed4931fa404f6983daa85b4c4ba2e2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6bed4931fa404f6983daa85b4c4ba2e2 2024-12-04T06:43:06,222 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/69fbd7af7d4b4198be033ba6f2207614 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/69fbd7af7d4b4198be033ba6f2207614 2024-12-04T06:43:06,226 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/A of 78d54f8244a2fd6e93eab21d996d309d into 69fbd7af7d4b4198be033ba6f2207614(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:06,226 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/B of 78d54f8244a2fd6e93eab21d996d309d into 6bed4931fa404f6983daa85b4c4ba2e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:06,226 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/A, priority=13, startTime=1733294585807; duration=0sec 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:06,226 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/B, priority=13, startTime=1733294585807; duration=0sec 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:A 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:B 2024-12-04T06:43:06,226 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:06,227 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:06,227 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 78d54f8244a2fd6e93eab21d996d309d/C is initiating minor compaction (all files) 2024-12-04T06:43:06,227 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78d54f8244a2fd6e93eab21d996d309d/C in TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:06,227 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b1fdbc0859da4488965adaf2ac29dc29, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp, totalSize=36.7 K 2024-12-04T06:43:06,228 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fdbc0859da4488965adaf2ac29dc29, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733294580771 2024-12-04T06:43:06,228 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-12-04T06:43:06,228 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:06,228 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,228 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:06,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:06,228 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f675692c2784f20869d365c4b7bc7a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733294581977 2024-12-04T06:43:06,228 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d999c3ee9eb49aa8979ddeb24b8f9ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733294584175 2024-12-04T06:43:06,233 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78d54f8244a2fd6e93eab21d996d309d#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:06,234 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d9e87264162749249b5c0b48dfb3d69c is 50, key is test_row_0/C:col10/1733294584194/Put/seqid=0 2024-12-04T06:43:06,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742352_1528 (size=13119) 2024-12-04T06:43:06,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/340fc688d46b49c282efe08cfa39c7a4 2024-12-04T06:43:06,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2fc78d29de0e4748934ea2c9b66031c5 is 50, key is test_row_0/B:col10/1733294586027/Put/seqid=0 2024-12-04T06:43:06,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742353_1529 (size=12301) 2024-12-04T06:43:06,641 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/d9e87264162749249b5c0b48dfb3d69c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d9e87264162749249b5c0b48dfb3d69c 2024-12-04T06:43:06,644 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78d54f8244a2fd6e93eab21d996d309d/C of 78d54f8244a2fd6e93eab21d996d309d into d9e87264162749249b5c0b48dfb3d69c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:06,644 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:06,644 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., storeName=78d54f8244a2fd6e93eab21d996d309d/C, priority=13, startTime=1733294585807; duration=0sec 2024-12-04T06:43:06,644 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:06,644 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78d54f8244a2fd6e93eab21d996d309d:C 2024-12-04T06:43:06,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2fc78d29de0e4748934ea2c9b66031c5 2024-12-04T06:43:06,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cf839bbaf83a46088ce8dd8753fb4907 is 50, key is test_row_0/C:col10/1733294586027/Put/seqid=0 2024-12-04T06:43:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742354_1530 (size=12301) 2024-12-04T06:43:07,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cf839bbaf83a46088ce8dd8753fb4907 2024-12-04T06:43:07,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/340fc688d46b49c282efe08cfa39c7a4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/340fc688d46b49c282efe08cfa39c7a4 2024-12-04T06:43:07,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/340fc688d46b49c282efe08cfa39c7a4, entries=150, sequenceid=336, filesize=12.0 K 2024-12-04T06:43:07,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/2fc78d29de0e4748934ea2c9b66031c5 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2fc78d29de0e4748934ea2c9b66031c5 2024-12-04T06:43:07,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2fc78d29de0e4748934ea2c9b66031c5, entries=150, sequenceid=336, filesize=12.0 K 2024-12-04T06:43:07,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/cf839bbaf83a46088ce8dd8753fb4907 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cf839bbaf83a46088ce8dd8753fb4907 2024-12-04T06:43:07,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cf839bbaf83a46088ce8dd8753fb4907, entries=150, sequenceid=336, filesize=12.0 K 2024-12-04T06:43:07,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 78d54f8244a2fd6e93eab21d996d309d in 1238ms, sequenceid=336, compaction requested=false 2024-12-04T06:43:07,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78d54f8244a2fd6e93eab21d996d309d: 2024-12-04T06:43:15,018 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-12-04T06:43:15,018 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:15,018 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:15,018 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:15,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:15,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:43:15,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1038 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3114 rows 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1039 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3113 rows 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1039 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3117 rows 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3136 rows 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1042 2024-12-04T06:43:15,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3120 rows 2024-12-04T06:43:15,019 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:43:15,019 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:43:15,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:15,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:15,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:15,020 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:43:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:15,023 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294595022"}]},"ts":"1733294595022"} 2024-12-04T06:43:15,024 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:43:15,024 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:43:15,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:43:15,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, UNASSIGN}] 2024-12-04T06:43:15,026 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, UNASSIGN 2024-12-04T06:43:15,026 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=78d54f8244a2fd6e93eab21d996d309d, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:15,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, UNASSIGN because future has completed 2024-12-04T06:43:15,028 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:43:15,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:15,180 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 78d54f8244a2fd6e93eab21d996d309d, disabling compactions & flushes 2024-12-04T06:43:15,180 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. after waiting 0 ms 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:15,180 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 78d54f8244a2fd6e93eab21d996d309d 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=A 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:15,180 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=B 2024-12-04T06:43:15,181 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:15,181 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78d54f8244a2fd6e93eab21d996d309d, store=C 2024-12-04T06:43:15,181 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:15,183 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a11e8a2735bb4fc59febf265163f14de is 50, key is test_row_0/A:col10/1733294586227/Put/seqid=0 2024-12-04T06:43:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742355_1531 (size=9857) 2024-12-04T06:43:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:15,587 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a11e8a2735bb4fc59febf265163f14de 2024-12-04T06:43:15,592 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/6c635a6666db42e4a728882cec6c0c30 is 50, key is test_row_0/B:col10/1733294586227/Put/seqid=0 2024-12-04T06:43:15,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742356_1532 (size=9857) 2024-12-04T06:43:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:15,996 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/6c635a6666db42e4a728882cec6c0c30 2024-12-04T06:43:16,001 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/b50f902bc0464884b2a2e0640c6a156b is 50, key is test_row_0/C:col10/1733294586227/Put/seqid=0 2024-12-04T06:43:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742357_1533 (size=9857) 2024-12-04T06:43:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:16,404 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/b50f902bc0464884b2a2e0640c6a156b 2024-12-04T06:43:16,408 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/A/a11e8a2735bb4fc59febf265163f14de as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a11e8a2735bb4fc59febf265163f14de 2024-12-04T06:43:16,410 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a11e8a2735bb4fc59febf265163f14de, entries=100, sequenceid=344, filesize=9.6 K 2024-12-04T06:43:16,411 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/B/6c635a6666db42e4a728882cec6c0c30 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6c635a6666db42e4a728882cec6c0c30 2024-12-04T06:43:16,413 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6c635a6666db42e4a728882cec6c0c30, entries=100, sequenceid=344, filesize=9.6 K 2024-12-04T06:43:16,413 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/.tmp/C/b50f902bc0464884b2a2e0640c6a156b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b50f902bc0464884b2a2e0640c6a156b 2024-12-04T06:43:16,416 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b50f902bc0464884b2a2e0640c6a156b, entries=100, sequenceid=344, filesize=9.6 K 2024-12-04T06:43:16,416 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 78d54f8244a2fd6e93eab21d996d309d in 1236ms, sequenceid=344, compaction requested=true 2024-12-04T06:43:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/d890c6c4e52941db8e237f3164f2acc7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/ddc71c51eef84b0dbc2fb0f5c8d39164, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9eb23bd6b83b498e9a342a43c10c27b0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/2b08f83d121e4b2aa28888a8a8e484ed, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f61bd12a1dbf4892ac4aad87459a765f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8d2a5cad483c416890e12d830f070128, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3] to archive 2024-12-04T06:43:16,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f3fa493a44744d94a1bb6b78fcdf15a2 2024-12-04T06:43:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/1877a36f24fb43679d238d0deef97ae5 2024-12-04T06:43:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/7097b96e54044081b9bfaaf3ad71c2c8 2024-12-04T06:43:16,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/b8caaf923efd4a70a8eedfc6b5429016 2024-12-04T06:43:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/d890c6c4e52941db8e237f3164f2acc7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/d890c6c4e52941db8e237f3164f2acc7 2024-12-04T06:43:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c4ad62bd04de4c8b8cf0056a530ca8df 2024-12-04T06:43:16,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/745f5b85e5044736a816ceeeca54249a 2024-12-04T06:43:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/ddc71c51eef84b0dbc2fb0f5c8d39164 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/ddc71c51eef84b0dbc2fb0f5c8d39164 2024-12-04T06:43:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/fe69528ad3214a21b17e46e0c8b9e857 2024-12-04T06:43:16,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/27e2283c75214307870e16e840a6ddc9 2024-12-04T06:43:16,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9eb23bd6b83b498e9a342a43c10c27b0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9eb23bd6b83b498e9a342a43c10c27b0 2024-12-04T06:43:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a2c550bd7e144b549b83067d335ff4b3 2024-12-04T06:43:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/2b08f83d121e4b2aa28888a8a8e484ed to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/2b08f83d121e4b2aa28888a8a8e484ed 2024-12-04T06:43:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/3a354b3aac0246928fd52135e6c51391 2024-12-04T06:43:16,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e01dad55793d4a56905014e5989c65f4 2024-12-04T06:43:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8114117b428e4fb7b6d506b3ad63bcd8 2024-12-04T06:43:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/e86ed0b8c74c4847ae3a99fa02110b8a 2024-12-04T06:43:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a7a1770f325a4f96b4c86654abc476b6 2024-12-04T06:43:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f61bd12a1dbf4892ac4aad87459a765f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/f61bd12a1dbf4892ac4aad87459a765f 2024-12-04T06:43:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/24e8315dfb6f4657bbe2edc79a674f54 2024-12-04T06:43:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/9c4adb41517c4cdcaf802dd390cffa72 2024-12-04T06:43:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8d2a5cad483c416890e12d830f070128 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/8d2a5cad483c416890e12d830f070128 2024-12-04T06:43:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/988ce54ccced43f6b3a3bfcaee28e62c 2024-12-04T06:43:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/c572dbcc05b745b2992a4269f6fc5cb3 2024-12-04T06:43:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/9e7297cfeb534de09f49a8f832e5fbf8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/cb6129d2f0c84023a294b4da00d505e2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dd101d547e694181bc0fca7c00ec7335, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3cf0ca737b4c484ca6aeebd32d1c6f0e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3437dda9a32547428b02234b5cceb17b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/0c1c8455a9654738b0a5a01995354b21, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b] to archive 2024-12-04T06:43:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/1e34c7339e34437ea79b43b8eb9792d1 2024-12-04T06:43:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/37d3b0ecdfc9426bb90d4b355c0ea000 2024-12-04T06:43:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/4228c3a51bf14ea6a237d84840eb4d0f 2024-12-04T06:43:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/9e7297cfeb534de09f49a8f832e5fbf8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/9e7297cfeb534de09f49a8f832e5fbf8 2024-12-04T06:43:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/05a7cfceb9854856a91a9b0da572d8f1 2024-12-04T06:43:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3808dc8c57a34c39a631381697aa59ab 2024-12-04T06:43:16,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/cb6129d2f0c84023a294b4da00d505e2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/cb6129d2f0c84023a294b4da00d505e2 2024-12-04T06:43:16,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/f5349ec3e65144e781a2925dcfde7f72 2024-12-04T06:43:16,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/123bbccc6b6e4cc18d7fb95e193fb3f5 2024-12-04T06:43:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dd101d547e694181bc0fca7c00ec7335 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dd101d547e694181bc0fca7c00ec7335 2024-12-04T06:43:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/210aad7ff33b4ee1a892ff6d7bdafc6c 2024-12-04T06:43:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3e4d217f94dd4e34970add638c5968ae 2024-12-04T06:43:16,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3cf0ca737b4c484ca6aeebd32d1c6f0e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3cf0ca737b4c484ca6aeebd32d1c6f0e 2024-12-04T06:43:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/dc991bf5b76e4f03a3c3b0d579803081 2024-12-04T06:43:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/b833e25202544fb0bf307afa45cf901b 2024-12-04T06:43:16,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/167fe31a198444469bd89d9edad4a9bb 2024-12-04T06:43:16,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/5b4a8143f3cb4aa282cb0c5d78b33355 2024-12-04T06:43:16,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3437dda9a32547428b02234b5cceb17b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3437dda9a32547428b02234b5cceb17b 2024-12-04T06:43:16,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2bbf941afe9843cb89879707b585b5d2 2024-12-04T06:43:16,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/43a0a038ceb145f6b11dfba5353f2cd3 2024-12-04T06:43:16,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/0c1c8455a9654738b0a5a01995354b21 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/0c1c8455a9654738b0a5a01995354b21 2024-12-04T06:43:16,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/ab91ed04f7b64046acb957a277517776 2024-12-04T06:43:16,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2c01b14a092d4e478a0886c16e1f2424 2024-12-04T06:43:16,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/3ae3afa66dc04aa8a37c302f25d0fc5b 2024-12-04T06:43:16,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/9590b0a24d1b460295bdaf9008a4332f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d35c6c4192454723a57714454ea8816f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/e27604925cb9407da6373b8fb3f63a4b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/dcac359e1b184a09aa0fe6cc30601677, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5b52a0d8b5ba4870875e1bc785cfefd4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b1fdbc0859da4488965adaf2ac29dc29, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca] to archive 2024-12-04T06:43:16,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:16,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d0c0d0ff2b0d42e882df21fded5710d1 2024-12-04T06:43:16,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/91a5ee9a241042b4aed0c9010ca70804 2024-12-04T06:43:16,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2958d7c74b46493c80faa63f8432eb35 2024-12-04T06:43:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/9590b0a24d1b460295bdaf9008a4332f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/9590b0a24d1b460295bdaf9008a4332f 2024-12-04T06:43:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f985330c037f4b82bdb15aebff846533 2024-12-04T06:43:16,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/19c2147ba1bf474e9c2cf3235b3e94c8 2024-12-04T06:43:16,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d35c6c4192454723a57714454ea8816f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d35c6c4192454723a57714454ea8816f 2024-12-04T06:43:16,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cc3b92c0d12747dc9672121fbc616e4a 2024-12-04T06:43:16,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/41b842e16e82454da6ebba1ac0bba146 2024-12-04T06:43:16,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/e27604925cb9407da6373b8fb3f63a4b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/e27604925cb9407da6373b8fb3f63a4b 2024-12-04T06:43:16,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/2f24ff7cb1274184b4ceeffc5b04add1 2024-12-04T06:43:16,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/53c3b33c83dc4a41a3f7a41562878c7b 2024-12-04T06:43:16,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/dcac359e1b184a09aa0fe6cc30601677 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/dcac359e1b184a09aa0fe6cc30601677 2024-12-04T06:43:16,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/872ecdb58eeb47fb8f9f0432b39b9a26 2024-12-04T06:43:16,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/33b32b2f756846a994adea41f49adfd6 2024-12-04T06:43:16,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/140611ba7e144f4eb926696d5b491bcc 2024-12-04T06:43:16,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f77780af848e4c6c9515524f22a11af0 2024-12-04T06:43:16,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5b52a0d8b5ba4870875e1bc785cfefd4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5b52a0d8b5ba4870875e1bc785cfefd4 2024-12-04T06:43:16,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/5e891eb1b5274b80955d5fea78df48de 2024-12-04T06:43:16,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/abbe2d40f84340f894a80d831d9aae4d 2024-12-04T06:43:16,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b1fdbc0859da4488965adaf2ac29dc29 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b1fdbc0859da4488965adaf2ac29dc29 2024-12-04T06:43:16,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/f1b10a9183444741b9a6e437706eabbd 2024-12-04T06:43:16,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7f675692c2784f20869d365c4b7bc7a4 2024-12-04T06:43:16,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/7d999c3ee9eb49aa8979ddeb24b8f9ca 2024-12-04T06:43:16,478 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=1 2024-12-04T06:43:16,478 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. 2024-12-04T06:43:16,479 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 78d54f8244a2fd6e93eab21d996d309d: Waiting for close lock at 1733294595180Running coprocessor pre-close hooks at 1733294595180Disabling compacts and flushes for region at 1733294595180Disabling writes for close at 1733294595180Obtaining lock to block concurrent updates at 1733294595180Preparing flush snapshotting stores in 78d54f8244a2fd6e93eab21d996d309d at 1733294595180Finished memstore snapshotting TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733294595181 (+1 ms)Flushing stores of TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d. at 1733294595181Flushing 78d54f8244a2fd6e93eab21d996d309d/A: creating writer at 1733294595181Flushing 78d54f8244a2fd6e93eab21d996d309d/A: appending metadata at 1733294595183 (+2 ms)Flushing 78d54f8244a2fd6e93eab21d996d309d/A: closing flushed file at 1733294595183Flushing 78d54f8244a2fd6e93eab21d996d309d/B: creating writer at 1733294595590 (+407 ms)Flushing 78d54f8244a2fd6e93eab21d996d309d/B: appending metadata at 1733294595592 (+2 ms)Flushing 78d54f8244a2fd6e93eab21d996d309d/B: closing flushed file at 1733294595592Flushing 78d54f8244a2fd6e93eab21d996d309d/C: creating writer at 1733294595999 (+407 ms)Flushing 78d54f8244a2fd6e93eab21d996d309d/C: appending metadata at 1733294596000 (+1 ms)Flushing 78d54f8244a2fd6e93eab21d996d309d/C: closing flushed file at 1733294596001 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58fcce2e: reopening flushed file at 1733294596407 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d54dd38: reopening flushed file at 1733294596410 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@205c2c72: reopening flushed file at 1733294596413 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 78d54f8244a2fd6e93eab21d996d309d in 1236ms, sequenceid=344, compaction requested=true at 1733294596416 (+3 ms)Writing region close event to WAL at 1733294596476 (+60 ms)Running coprocessor post-close hooks at 1733294596478 (+2 ms)Closed at 1733294596478 2024-12-04T06:43:16,480 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:16,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=78d54f8244a2fd6e93eab21d996d309d, regionState=CLOSED 2024-12-04T06:43:16,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:43:16,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-04T06:43:16,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 78d54f8244a2fd6e93eab21d996d309d, server=607fd5c6574c,38257,1733294430862 in 1.4550 sec 2024-12-04T06:43:16,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-04T06:43:16,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78d54f8244a2fd6e93eab21d996d309d, UNASSIGN in 1.4590 sec 2024-12-04T06:43:16,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T06:43:16,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4620 sec 2024-12-04T06:43:16,488 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294596487"}]},"ts":"1733294596487"} 2024-12-04T06:43:16,489 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:43:16,489 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:43:16,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4690 sec 2024-12-04T06:43:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T06:43:17,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:17,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:43:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,168 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T06:43:17,169 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,172 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:17,173 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/recovered.edits] 2024-12-04T06:43:17,175 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/340fc688d46b49c282efe08cfa39c7a4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/340fc688d46b49c282efe08cfa39c7a4 2024-12-04T06:43:17,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/69fbd7af7d4b4198be033ba6f2207614 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/69fbd7af7d4b4198be033ba6f2207614 2024-12-04T06:43:17,177 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a11e8a2735bb4fc59febf265163f14de to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/A/a11e8a2735bb4fc59febf265163f14de 2024-12-04T06:43:17,178 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2fc78d29de0e4748934ea2c9b66031c5 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/2fc78d29de0e4748934ea2c9b66031c5 2024-12-04T06:43:17,179 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6bed4931fa404f6983daa85b4c4ba2e2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6bed4931fa404f6983daa85b4c4ba2e2 2024-12-04T06:43:17,180 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6c635a6666db42e4a728882cec6c0c30 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/B/6c635a6666db42e4a728882cec6c0c30 2024-12-04T06:43:17,181 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b50f902bc0464884b2a2e0640c6a156b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/b50f902bc0464884b2a2e0640c6a156b 2024-12-04T06:43:17,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cf839bbaf83a46088ce8dd8753fb4907 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/cf839bbaf83a46088ce8dd8753fb4907 2024-12-04T06:43:17,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d9e87264162749249b5c0b48dfb3d69c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/C/d9e87264162749249b5c0b48dfb3d69c 2024-12-04T06:43:17,184 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/recovered.edits/347.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d/recovered.edits/347.seqid 2024-12-04T06:43:17,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/78d54f8244a2fd6e93eab21d996d309d 2024-12-04T06:43:17,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:43:17,186 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,188 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:43:17,189 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:43:17,190 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,190 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:43:17,190 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294597190"}]},"ts":"9223372036854775807"} 2024-12-04T06:43:17,192 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:43:17,192 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 78d54f8244a2fd6e93eab21d996d309d, NAME => 'TestAcidGuarantees,,1733294563366.78d54f8244a2fd6e93eab21d996d309d.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:43:17,192 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:43:17,192 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294597192"}]},"ts":"9223372036854775807"} 2024-12-04T06:43:17,193 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:43:17,194 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-12-04T06:43:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T06:43:17,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:43:17,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:17,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=441 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 370), ProcessCount=11 (was 11), AvailableMemoryMB=5837 (was 5895) 2024-12-04T06:43:17,294 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=5836 2024-12-04T06:43:17,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:43:17,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:43:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:17,297 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T06:43:17,297 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:17,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-04T06:43:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:17,297 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T06:43:17,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742358_1534 (size=1061) 2024-12-04T06:43:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:17,703 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d 2024-12-04T06:43:17,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742359_1535 (size=53) 2024-12-04T06:43:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1aa8c1f4b46ff15f781ef8f5e098cbaa, disabling compactions & flushes 2024-12-04T06:43:18,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. after waiting 0 ms 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: Waiting for close lock at 1733294598108Disabling compacts and flushes for region at 1733294598108Disabling writes for close at 1733294598108Writing region close event to WAL at 1733294598108Closed at 1733294598108 2024-12-04T06:43:18,109 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T06:43:18,109 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733294598109"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733294598109"}]},"ts":"1733294598109"} 2024-12-04T06:43:18,111 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T06:43:18,111 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T06:43:18,111 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294598111"}]},"ts":"1733294598111"} 2024-12-04T06:43:18,112 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-04T06:43:18,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, ASSIGN}] 2024-12-04T06:43:18,113 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, ASSIGN 2024-12-04T06:43:18,114 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, ASSIGN; state=OFFLINE, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=false 2024-12-04T06:43:18,264 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:18,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, ASSIGN because future has completed 2024-12-04T06:43:18,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:43:18,420 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,420 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:43:18,420 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,420 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:43:18,420 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,421 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,421 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,422 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:18,423 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName A 2024-12-04T06:43:18,423 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:18,423 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:18,423 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,424 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:18,424 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName B 2024-12-04T06:43:18,424 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:18,424 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:18,424 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,425 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:18,425 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName C 2024-12-04T06:43:18,425 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:18,425 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:18,425 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,426 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,426 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,426 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,427 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,427 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,427 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:43:18,428 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,430 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T06:43:18,430 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 1aa8c1f4b46ff15f781ef8f5e098cbaa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59595735, jitterRate=-0.11195434629917145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:43:18,430 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:18,431 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: Running coprocessor pre-open hook at 1733294598421Writing region info on filesystem at 1733294598421Initializing all the Stores at 1733294598421Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294598421Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294598421Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294598421Cleaning up temporary data from old regions at 1733294598427 (+6 ms)Running coprocessor post-open hooks at 1733294598430 (+3 ms)Region opened successfully at 1733294598431 (+1 ms) 2024-12-04T06:43:18,431 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., pid=140, masterSystemTime=1733294598418 2024-12-04T06:43:18,432 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,433 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:18,433 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=OPEN, openSeqNum=2, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:18,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:43:18,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-04T06:43:18,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 in 169 msec 2024-12-04T06:43:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:18,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-04T06:43:18,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, ASSIGN in 324 msec 2024-12-04T06:43:18,438 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T06:43:18,438 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294598438"}]},"ts":"1733294598438"} 2024-12-04T06:43:18,439 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-04T06:43:18,440 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T06:43:18,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-12-04T06:43:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-04T06:43:19,447 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:19,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bdefb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:19,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:19,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:19,449 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9cddc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:19,450 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a11bf84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:19,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:19,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:19,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T06:43:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 607fd5c6574c,45619,1733294430018 2024-12-04T06:43:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ae4506 2024-12-04T06:43:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T06:43:19,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T06:43:19,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-04T06:43:19,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T06:43:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:19,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742360_1536 (size=1097) 2024-12-04T06:43:19,865 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-04T06:43:19,865 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-04T06:43:19,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:43:19,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE}] 2024-12-04T06:43:19,869 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE 2024-12-04T06:43:19,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:19,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE because future has completed 2024-12-04T06:43:19,871 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:43:19,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:43:20,023 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,023 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:43:20,023 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 1aa8c1f4b46ff15f781ef8f5e098cbaa, disabling compactions & flushes 2024-12-04T06:43:20,023 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,023 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,023 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. after waiting 0 ms 2024-12-04T06:43:20,023 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,026 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-04T06:43:20,027 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,027 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: Waiting for close lock at 1733294600023Running coprocessor pre-close hooks at 1733294600023Disabling compacts and flushes for region at 1733294600023Disabling writes for close at 1733294600023Writing region close event to WAL at 1733294600024 (+1 ms)Running coprocessor post-close hooks at 1733294600026 (+2 ms)Closed at 1733294600026 2024-12-04T06:43:20,027 WARN [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 1aa8c1f4b46ff15f781ef8f5e098cbaa to self. 2024-12-04T06:43:20,028 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=CLOSED 2024-12-04T06:43:20,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:43:20,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-04T06:43:20,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 in 159 msec 2024-12-04T06:43:20,031 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE; state=CLOSED, location=607fd5c6574c,38257,1733294430862; forceNewPlan=false, retain=true 2024-12-04T06:43:20,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=OPENING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE because future has completed 2024-12-04T06:43:20,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:43:20,337 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,338 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} 2024-12-04T06:43:20,338 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,338 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T06:43:20,338 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,338 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,339 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,340 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:20,340 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName A 2024-12-04T06:43:20,341 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:20,341 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:20,341 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,342 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:20,342 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName B 2024-12-04T06:43:20,342 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:20,342 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:20,342 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,343 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-04T06:43:20,343 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1aa8c1f4b46ff15f781ef8f5e098cbaa columnFamilyName C 2024-12-04T06:43:20,343 DEBUG [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:20,343 INFO [StoreOpener-1aa8c1f4b46ff15f781ef8f5e098cbaa-1 {}] regionserver.HStore(327): Store=1aa8c1f4b46ff15f781ef8f5e098cbaa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T06:43:20,343 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,343 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,344 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,344 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,345 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,345 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,345 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T06:43:20,346 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,347 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 1aa8c1f4b46ff15f781ef8f5e098cbaa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71582762, jitterRate=0.06666627526283264}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T06:43:20,347 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,347 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: Running coprocessor pre-open hook at 1733294600338Writing region info on filesystem at 1733294600338Initializing all the Stores at 1733294600339 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294600339Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294600339Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733294600339Cleaning up temporary data from old regions at 1733294600345 (+6 ms)Running coprocessor post-open hooks at 1733294600347 (+2 ms)Region opened successfully at 1733294600347 2024-12-04T06:43:20,348 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., pid=145, masterSystemTime=1733294600335 2024-12-04T06:43:20,349 DEBUG [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,349 INFO [RS_OPEN_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,350 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=OPEN, openSeqNum=5, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:43:20,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-04T06:43:20,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 in 168 msec 2024-12-04T06:43:20,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-04T06:43:20,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, REOPEN/MOVE in 484 msec 2024-12-04T06:43:20,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T06:43:20,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-12-04T06:43:20,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-12-04T06:43:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-04T06:43:20,358 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:20,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdaa2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,360 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@198eedb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,361 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117d0acb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ecb8793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,363 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e130a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,364 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1a2be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccad3c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,365 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d57c58a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,366 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7f6276, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74abaaed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,368 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66375d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e11e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55149d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692c5327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc59369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a0c8e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,373 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed09a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618f081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3499f302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42995279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,376 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1525bf5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1675bdc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,378 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b80701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,379 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da9b770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78526251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1d80cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b47b245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc4680c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 607fd5c6574c,45619,-1 for getting cluster id 2024-12-04T06:43:20,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T06:43:20,382 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '75f52202-11fd-4086-b847-f37b0601f8c4' 2024-12-04T06:43:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T06:43:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "75f52202-11fd-4086-b847-f37b0601f8c4" 2024-12-04T06:43:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713811c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [607fd5c6574c,45619,-1] 2024-12-04T06:43:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T06:43:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:20,383 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T06:43:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f18f8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T06:43:20,384 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,384 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,384 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,384 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,385 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,386 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,386 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,386 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,386 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,387 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:20,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,388 DEBUG [Thread-2402 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,388 DEBUG [Thread-2402 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=607fd5c6574c,38257,1733294430862, seqNum=-1] 2024-12-04T06:43:20,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T06:43:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-04T06:43:20,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T06:43:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,390 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:20,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:20,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5] 2024-12-04T06:43:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:20,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:20,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465891534cfff4afea7d903390305640d_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294600395/Put/seqid=0 2024-12-04T06:43:20,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742361_1537 (size=9714) 2024-12-04T06:43:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294660476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294660477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294660480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294600473, max timestamp=1733294600482Num uniques 859; 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294600473, max timestamp=1733294600482Num uniques 859; 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294600473, max timestamp=1733294600482Num uniques 859; 2024-12-04T06:43:20,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294660514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294660519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:20,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:20,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294660587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294660596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294660596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294660627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294660627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:20,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294660796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294660806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294660806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294660837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294660837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:20,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:20,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:20,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:20,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465891534cfff4afea7d903390305640d_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465891534cfff4afea7d903390305640d_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:20,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/5c0203c4774d440f8bfb32b7bc699e19, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:20,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/5c0203c4774d440f8bfb32b7bc699e19 is 175, key is test_row_0/A:col10/1733294600395/Put/seqid=0 2024-12-04T06:43:20,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742362_1538 (size=22361) 2024-12-04T06:43:20,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/5c0203c4774d440f8bfb32b7bc699e19 2024-12-04T06:43:20,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/b6981160b0bf4718b9f9f1ace55d373c is 50, key is test_row_0/B:col10/1733294600395/Put/seqid=0 2024-12-04T06:43:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742363_1539 (size=9657) 2024-12-04T06:43:20,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/b6981160b0bf4718b9f9f1ace55d373c 2024-12-04T06:43:20,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/c22fb14066a8449ab1cbd973df12795a is 50, key is test_row_0/C:col10/1733294600395/Put/seqid=0 2024-12-04T06:43:20,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742364_1540 (size=9657) 2024-12-04T06:43:21,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:21,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:21,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294661106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294661116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294661116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294661147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294661147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:21,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/c22fb14066a8449ab1cbd973df12795a 2024-12-04T06:43:21,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/5c0203c4774d440f8bfb32b7bc699e19 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19 2024-12-04T06:43:21,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19, entries=100, sequenceid=15, filesize=21.8 K 2024-12-04T06:43:21,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/b6981160b0bf4718b9f9f1ace55d373c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c 2024-12-04T06:43:21,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c, entries=100, sequenceid=15, filesize=9.4 K 2024-12-04T06:43:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/c22fb14066a8449ab1cbd973df12795a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a 2024-12-04T06:43:21,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a, entries=100, sequenceid=15, filesize=9.4 K 2024-12-04T06:43:21,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 902ms, sequenceid=15, compaction requested=false 2024-12-04T06:43:21,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:21,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-04T06:43:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:21,464 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:21,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:21,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b31a2ae85a5943f3a26eb64407065b35_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294600513/Put/seqid=0 2024-12-04T06:43:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742365_1541 (size=12154) 2024-12-04T06:43:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:21,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294661636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294661638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294661638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294661656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294661656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294661747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294661747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294661747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:21,879 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b31a2ae85a5943f3a26eb64407065b35_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b31a2ae85a5943f3a26eb64407065b35_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ba909a5870d3456b864dee1467f310a3, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ba909a5870d3456b864dee1467f310a3 is 175, key is test_row_0/A:col10/1733294600513/Put/seqid=0 2024-12-04T06:43:21,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742366_1542 (size=30955) 2024-12-04T06:43:21,884 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ba909a5870d3456b864dee1467f310a3 2024-12-04T06:43:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/0b6092f0c56541d8b52df26f6e147f28 is 50, key is test_row_0/B:col10/1733294600513/Put/seqid=0 2024-12-04T06:43:21,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742367_1543 (size=12001) 2024-12-04T06:43:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294661957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294661957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294661957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:21,995 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T06:43:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294662266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294662277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294662277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,293 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/0b6092f0c56541d8b52df26f6e147f28 2024-12-04T06:43:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6eaeec6e2beb4257bd33f4726e3cd696 is 50, key is test_row_0/C:col10/1733294600513/Put/seqid=0 2024-12-04T06:43:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742368_1544 (size=12001) 2024-12-04T06:43:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294662676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294662676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,680 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,681 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,705 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6eaeec6e2beb4257bd33f4726e3cd696 2024-12-04T06:43:22,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ba909a5870d3456b864dee1467f310a3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3 2024-12-04T06:43:22,712 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3, entries=150, sequenceid=50, filesize=30.2 K 2024-12-04T06:43:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/0b6092f0c56541d8b52df26f6e147f28 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28 2024-12-04T06:43:22,715 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T06:43:22,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6eaeec6e2beb4257bd33f4726e3cd696 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696 2024-12-04T06:43:22,719 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696, entries=150, sequenceid=50, filesize=11.7 K 2024-12-04T06:43:22,720 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1256ms, sequenceid=50, compaction requested=false 2024-12-04T06:43:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-04T06:43:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-04T06:43:22,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-04T06:43:22,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-12-04T06:43:22,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.3360 sec 2024-12-04T06:43:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:22,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:22,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418603e9de38246e99b1bee6489490798_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742369_1545 (size=14594) 2024-12-04T06:43:22,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294662813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294662814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294662816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294662927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294662927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294662927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294663137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294663137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294663137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,194 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:23,197 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120418603e9de38246e99b1bee6489490798_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418603e9de38246e99b1bee6489490798_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:23,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/2d9c0ddf899046e0bd004e961041113e, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:23,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/2d9c0ddf899046e0bd004e961041113e is 175, key is test_row_0/A:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742370_1546 (size=39549) 2024-12-04T06:43:23,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/2d9c0ddf899046e0bd004e961041113e 2024-12-04T06:43:23,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3bcd37d27f7d44388438fd9943a71222 is 50, key is test_row_0/B:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:23,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742371_1547 (size=12001) 2024-12-04T06:43:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294663447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294663447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294663447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3bcd37d27f7d44388438fd9943a71222 2024-12-04T06:43:23,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/25014f33e76d4500843138871fb3d273 is 50, key is test_row_0/C:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:23,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742372_1548 (size=12001) 2024-12-04T06:43:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294663957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294663957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294663966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/25014f33e76d4500843138871fb3d273 2024-12-04T06:43:24,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/2d9c0ddf899046e0bd004e961041113e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e 2024-12-04T06:43:24,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e, entries=200, sequenceid=63, filesize=38.6 K 2024-12-04T06:43:24,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3bcd37d27f7d44388438fd9943a71222 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222 2024-12-04T06:43:24,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222, entries=150, sequenceid=63, filesize=11.7 K 2024-12-04T06:43:24,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/25014f33e76d4500843138871fb3d273 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273 2024-12-04T06:43:24,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273, entries=150, sequenceid=63, filesize=11.7 K 2024-12-04T06:43:24,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1257ms, sequenceid=63, compaction requested=true 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:24,036 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:24,036 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:24,037 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:24,037 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/A is initiating minor compaction (all files) 2024-12-04T06:43:24,037 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/A in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,037 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=90.7 K 2024-12-04T06:43:24,037 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,037 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e] 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c0203c4774d440f8bfb32b7bc699e19, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294600395 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/B is initiating minor compaction (all files) 2024-12-04T06:43:24,038 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/B in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,038 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=32.9 K 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba909a5870d3456b864dee1467f310a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733294600509 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b6981160b0bf4718b9f9f1ace55d373c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294600395 2024-12-04T06:43:24,038 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d9c0ddf899046e0bd004e961041113e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:24,039 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b6092f0c56541d8b52df26f6e147f28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733294600509 2024-12-04T06:43:24,039 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcd37d27f7d44388438fd9943a71222, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:24,043 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:24,045 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:24,046 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/9a02727d35a34eaba5b8c7d2ae363f69 is 50, key is test_row_0/B:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:24,047 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204fff93b14f0ec475cb1efd7154bba7b9b_1aa8c1f4b46ff15f781ef8f5e098cbaa store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:24,049 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204fff93b14f0ec475cb1efd7154bba7b9b_1aa8c1f4b46ff15f781ef8f5e098cbaa, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:24,049 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204fff93b14f0ec475cb1efd7154bba7b9b_1aa8c1f4b46ff15f781ef8f5e098cbaa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:24,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742373_1549 (size=12104) 2024-12-04T06:43:24,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742374_1550 (size=4469) 2024-12-04T06:43:24,470 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/9a02727d35a34eaba5b8c7d2ae363f69 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/9a02727d35a34eaba5b8c7d2ae363f69 2024-12-04T06:43:24,473 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/B of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 9a02727d35a34eaba5b8c7d2ae363f69(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:24,473 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:24,473 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/B, priority=13, startTime=1733294604036; duration=0sec 2024-12-04T06:43:24,473 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:24,473 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:B 2024-12-04T06:43:24,474 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:24,474 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:24,474 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/C is initiating minor compaction (all files) 2024-12-04T06:43:24,474 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/C in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,474 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=32.9 K 2024-12-04T06:43:24,475 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c22fb14066a8449ab1cbd973df12795a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733294600395 2024-12-04T06:43:24,475 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eaeec6e2beb4257bd33f4726e3cd696, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733294600509 2024-12-04T06:43:24,475 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 25014f33e76d4500843138871fb3d273, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:24,477 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#A#compaction#470 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:24,478 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f71197f54a1349e58211eed6dc834845 is 175, key is test_row_0/A:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742375_1551 (size=31058) 2024-12-04T06:43:24,482 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T06:43:24,483 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/5e0904eedf0d432b8a3d7260547bc6f8 is 50, key is test_row_0/C:col10/1733294602778/Put/seqid=0 2024-12-04T06:43:24,485 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f71197f54a1349e58211eed6dc834845 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845 2024-12-04T06:43:24,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742376_1552 (size=12104) 2024-12-04T06:43:24,491 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/A of 1aa8c1f4b46ff15f781ef8f5e098cbaa into f71197f54a1349e58211eed6dc834845(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:24,491 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:24,491 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/A, priority=13, startTime=1733294604036; duration=0sec 2024-12-04T06:43:24,491 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:24,491 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:A 2024-12-04T06:43:24,491 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/5e0904eedf0d432b8a3d7260547bc6f8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/5e0904eedf0d432b8a3d7260547bc6f8 2024-12-04T06:43:24,495 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/C of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 5e0904eedf0d432b8a3d7260547bc6f8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:24,495 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:24,495 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/C, priority=13, startTime=1733294604036; duration=0sec 2024-12-04T06:43:24,495 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:24,495 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:C 2024-12-04T06:43:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T06:43:24,557 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:24,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-04T06:43:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:24,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:24,560 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:24,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:24,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:24,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:24,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120469d12e39074a4d20a839562f217341cf_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294604688/Put/seqid=0 2024-12-04T06:43:24,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742377_1553 (size=14594) 2024-12-04T06:43:24,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T06:43:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294664715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294664716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294664826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294664826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T06:43:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:24,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:24,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294664967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294664967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,969 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,969 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294664976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:24,978 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_2' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T06:43:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:25,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294665037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294665037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:25,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120469d12e39074a4d20a839562f217341cf_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120469d12e39074a4d20a839562f217341cf_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:25,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/867869329c904325bfc3304e3adca55d, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:25,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/867869329c904325bfc3304e3adca55d is 175, key is test_row_0/A:col10/1733294604688/Put/seqid=0 2024-12-04T06:43:25,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742378_1554 (size=39549) 2024-12-04T06:43:25,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/867869329c904325bfc3304e3adca55d 2024-12-04T06:43:25,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/a53fec52e31048ebb4a94ce501becb84 is 50, key is test_row_0/B:col10/1733294604688/Put/seqid=0 2024-12-04T06:43:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742379_1555 (size=12001) 2024-12-04T06:43:25,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/a53fec52e31048ebb4a94ce501becb84 2024-12-04T06:43:25,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/d2a47f6cae214429875bcc763d33cfcc is 50, key is test_row_0/C:col10/1733294604688/Put/seqid=0 2024-12-04T06:43:25,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T06:43:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:25,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:25,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:25,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742380_1556 (size=12001) 2024-12-04T06:43:25,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/d2a47f6cae214429875bcc763d33cfcc 2024-12-04T06:43:25,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/867869329c904325bfc3304e3adca55d as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d 2024-12-04T06:43:25,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d, entries=200, sequenceid=86, filesize=38.6 K 2024-12-04T06:43:25,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/a53fec52e31048ebb4a94ce501becb84 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84 2024-12-04T06:43:25,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T06:43:25,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/d2a47f6cae214429875bcc763d33cfcc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc 2024-12-04T06:43:25,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc, entries=150, sequenceid=86, filesize=11.7 K 2024-12-04T06:43:25,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 526ms, sequenceid=86, compaction requested=false 2024-12-04T06:43:25,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:25,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-04T06:43:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:25,325 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:25,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045d6e75f3354b46658e34566a457bdeb5_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294604696/Put/seqid=0 2024-12-04T06:43:25,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742381_1557 (size=12154) 2024-12-04T06:43:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:25,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294665391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294665394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294665506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294665506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:25,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294665716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294665716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:25,746 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412045d6e75f3354b46658e34566a457bdeb5_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d6e75f3354b46658e34566a457bdeb5_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/b6521fb0c8d543a5a624375323821713, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/b6521fb0c8d543a5a624375323821713 is 175, key is test_row_0/A:col10/1733294604696/Put/seqid=0 2024-12-04T06:43:25,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742382_1558 (size=30955) 2024-12-04T06:43:25,774 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/b6521fb0c8d543a5a624375323821713 2024-12-04T06:43:25,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/fb08a430d4714a6081341559a089af59 is 50, key is test_row_0/B:col10/1733294604696/Put/seqid=0 2024-12-04T06:43:25,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742383_1559 (size=12001) 2024-12-04T06:43:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294666027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294666027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:26,242 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/fb08a430d4714a6081341559a089af59 2024-12-04T06:43:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/be95771682624aab92f73248ca946cb3 is 50, key is test_row_0/C:col10/1733294604696/Put/seqid=0 2024-12-04T06:43:26,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742384_1560 (size=12001) 2024-12-04T06:43:26,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294666537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:26,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294666547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:26,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:26,654 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/be95771682624aab92f73248ca946cb3 2024-12-04T06:43:26,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/b6521fb0c8d543a5a624375323821713 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713 2024-12-04T06:43:26,663 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713, entries=150, sequenceid=98, filesize=30.2 K 2024-12-04T06:43:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/fb08a430d4714a6081341559a089af59 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59 2024-12-04T06:43:26,669 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59, entries=150, sequenceid=98, filesize=11.7 K 2024-12-04T06:43:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/be95771682624aab92f73248ca946cb3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3 2024-12-04T06:43:26,677 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3, entries=150, sequenceid=98, filesize=11.7 K 2024-12-04T06:43:26,678 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1352ms, sequenceid=98, compaction requested=true 2024-12-04T06:43:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-04T06:43:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-04T06:43:26,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-04T06:43:26,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-12-04T06:43:26,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.1250 sec 2024-12-04T06:43:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T06:43:26,708 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:26,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-04T06:43:26,711 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:26,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:26,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T06:43:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T06:43:26,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:26,865 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:26,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b057697f390b4c01a2ce24c8a9d6b3d7_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294605390/Put/seqid=0 2024-12-04T06:43:26,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742385_1561 (size=12154) 2024-12-04T06:43:26,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:26,879 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b057697f390b4c01a2ce24c8a9d6b3d7_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b057697f390b4c01a2ce24c8a9d6b3d7_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:26,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f4081da702f540459d73f4f11393aa99, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f4081da702f540459d73f4f11393aa99 is 175, key is test_row_0/A:col10/1733294605390/Put/seqid=0 2024-12-04T06:43:26,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742386_1562 (size=30955) 2024-12-04T06:43:26,884 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f4081da702f540459d73f4f11393aa99 2024-12-04T06:43:26,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/c16236a282e146b9bbc05201f726e986 is 50, key is test_row_0/B:col10/1733294605390/Put/seqid=0 2024-12-04T06:43:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742387_1563 (size=12001) 2024-12-04T06:43:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294667001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294667001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294667002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T06:43:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294667116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294667117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294667117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,304 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/c16236a282e146b9bbc05201f726e986 2024-12-04T06:43:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/f667bf2ffa374fd38197c8ade3d80d04 is 50, key is test_row_0/C:col10/1733294605390/Put/seqid=0 2024-12-04T06:43:27,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742388_1564 (size=12001) 2024-12-04T06:43:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294667326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294667326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294667327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T06:43:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294667546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,550 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294667557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,561 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294667636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294667637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294667637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:27,715 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/f667bf2ffa374fd38197c8ade3d80d04 2024-12-04T06:43:27,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/f4081da702f540459d73f4f11393aa99 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99 2024-12-04T06:43:27,723 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99, entries=150, sequenceid=118, filesize=30.2 K 2024-12-04T06:43:27,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/c16236a282e146b9bbc05201f726e986 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986 2024-12-04T06:43:27,726 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986, entries=150, sequenceid=118, filesize=11.7 K 2024-12-04T06:43:27,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/f667bf2ffa374fd38197c8ade3d80d04 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04 2024-12-04T06:43:27,731 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04, entries=150, sequenceid=118, filesize=11.7 K 2024-12-04T06:43:27,731 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 866ms, sequenceid=118, compaction requested=true 2024-12-04T06:43:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:27,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-04T06:43:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-04T06:43:27,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-04T06:43:27,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-12-04T06:43:27,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0270 sec 2024-12-04T06:43:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-04T06:43:27,847 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:27,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-04T06:43:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:27,848 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:27,849 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:27,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:28,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-04T06:43:28,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:28,002 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:28,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120424650512c56e424baab256ad49a57779_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294607000/Put/seqid=0 2024-12-04T06:43:28,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742389_1565 (size=12204) 2024-12-04T06:43:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:28,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294668182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294668188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294668188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294668296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294668306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294668307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:28,416 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120424650512c56e424baab256ad49a57779_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120424650512c56e424baab256ad49a57779_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:28,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/c3d7cdfc3358466880128d95584fe5bc, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:28,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/c3d7cdfc3358466880128d95584fe5bc is 175, key is test_row_0/A:col10/1733294607000/Put/seqid=0 2024-12-04T06:43:28,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742390_1566 (size=31005) 2024-12-04T06:43:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294668506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294668516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294668517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294668817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,821 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/c3d7cdfc3358466880128d95584fe5bc 2024-12-04T06:43:28,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/5826536f39ff4069a2da2e794fe6cff6 is 50, key is test_row_0/B:col10/1733294607000/Put/seqid=0 2024-12-04T06:43:28,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742391_1567 (size=12051) 2024-12-04T06:43:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294668826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294668826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:29,081 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T06:43:29,231 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/5826536f39ff4069a2da2e794fe6cff6 2024-12-04T06:43:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/ed9b549765d74098b14377c68046bbd3 is 50, key is test_row_0/C:col10/1733294607000/Put/seqid=0 2024-12-04T06:43:29,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742392_1568 (size=12051) 2024-12-04T06:43:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294669327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:29,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294669337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:29,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294669337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294669567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:29,571 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:29,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294669587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:29,591 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:29,648 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/ed9b549765d74098b14377c68046bbd3 2024-12-04T06:43:29,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/c3d7cdfc3358466880128d95584fe5bc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc 2024-12-04T06:43:29,655 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc, entries=150, sequenceid=130, filesize=30.3 K 2024-12-04T06:43:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/5826536f39ff4069a2da2e794fe6cff6 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6 2024-12-04T06:43:29,658 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6, entries=150, sequenceid=130, filesize=11.8 K 2024-12-04T06:43:29,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/ed9b549765d74098b14377c68046bbd3 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3 2024-12-04T06:43:29,662 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3, entries=150, sequenceid=130, filesize=11.8 K 2024-12-04T06:43:29,662 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1661ms, sequenceid=130, compaction requested=true 2024-12-04T06:43:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:29,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-04T06:43:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-04T06:43:29,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-04T06:43:29,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-04T06:43:29,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8190 sec 2024-12-04T06:43:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-04T06:43:29,997 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:29,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-04T06:43:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:29,999 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:30,000 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:30,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:30,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-04T06:43:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:30,152 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:30,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:30,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e155436b1c4a4dff9021310558d84118_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294608179/Put/seqid=0 2024-12-04T06:43:30,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742393_1569 (size=12304) 2024-12-04T06:43:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:30,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294670360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294670361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294670362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294670477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294670477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294670477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:30,567 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e155436b1c4a4dff9021310558d84118_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e155436b1c4a4dff9021310558d84118_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3bf9995c630d40489dd86984ed7f73b7, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:30,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3bf9995c630d40489dd86984ed7f73b7 is 175, key is test_row_0/A:col10/1733294608179/Put/seqid=0 2024-12-04T06:43:30,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742394_1570 (size=31105) 2024-12-04T06:43:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294670687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294670687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294670687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,973 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3bf9995c630d40489dd86984ed7f73b7 2024-12-04T06:43:30,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3c5e423248fd4c7b9e2c99b1e47ee367 is 50, key is test_row_0/B:col10/1733294608179/Put/seqid=0 2024-12-04T06:43:30,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742395_1571 (size=12151) 2024-12-04T06:43:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294670996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294670996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:30,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294670997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:31,382 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3c5e423248fd4c7b9e2c99b1e47ee367 2024-12-04T06:43:31,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/95849e77d26a4963bd644c9192124e6c is 50, key is test_row_0/C:col10/1733294608179/Put/seqid=0 2024-12-04T06:43:31,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742396_1572 (size=12151) 2024-12-04T06:43:31,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294671506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:31,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294671507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:31,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294671507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:31,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:31,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:31,793 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/95849e77d26a4963bd644c9192124e6c 2024-12-04T06:43:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3bf9995c630d40489dd86984ed7f73b7 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7 2024-12-04T06:43:31,801 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7, entries=150, sequenceid=150, filesize=30.4 K 2024-12-04T06:43:31,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3c5e423248fd4c7b9e2c99b1e47ee367 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367 2024-12-04T06:43:31,805 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T06:43:31,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/95849e77d26a4963bd644c9192124e6c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c 2024-12-04T06:43:31,809 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c, entries=150, sequenceid=150, filesize=11.9 K 2024-12-04T06:43:31,810 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1658ms, sequenceid=150, compaction requested=true 2024-12-04T06:43:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-04T06:43:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-04T06:43:31,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-04T06:43:31,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-12-04T06:43:31,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8150 sec 2024-12-04T06:43:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-04T06:43:32,146 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:32,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-04T06:43:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:32,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:32,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:32,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:32,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-04T06:43:32,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:32,302 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204af7215c743a44715a945c209452437eb_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294610339/Put/seqid=0 2024-12-04T06:43:32,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742397_1573 (size=12304) 2024-12-04T06:43:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:32,315 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204af7215c743a44715a945c209452437eb_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204af7215c743a44715a945c209452437eb_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:32,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/04d39fbe462a41fda3a6482371155545, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/04d39fbe462a41fda3a6482371155545 is 175, key is test_row_0/A:col10/1733294610339/Put/seqid=0 2024-12-04T06:43:32,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742398_1574 (size=31105) 2024-12-04T06:43:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294672557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294672558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294672559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294672666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294672667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294672677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,698 DEBUG [regionserver/607fd5c6574c:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.23076923076923078, tune throughput to 61.54 MB/second 2024-12-04T06:43:32,720 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/04d39fbe462a41fda3a6482371155545 2024-12-04T06:43:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/86f40af27d314178a5f347f322172489 is 50, key is test_row_0/B:col10/1733294610339/Put/seqid=0 2024-12-04T06:43:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742399_1575 (size=12151) 2024-12-04T06:43:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:32,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294672877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294672877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:32,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294672887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,130 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/86f40af27d314178a5f347f322172489 2024-12-04T06:43:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/b3134c394d5f4e3d99107d08c0d79b6c is 50, key is test_row_0/C:col10/1733294610339/Put/seqid=0 2024-12-04T06:43:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742400_1576 (size=12151) 2024-12-04T06:43:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294673186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294673196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294673196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:33,540 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/b3134c394d5f4e3d99107d08c0d79b6c 2024-12-04T06:43:33,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/04d39fbe462a41fda3a6482371155545 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545 2024-12-04T06:43:33,548 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545, entries=150, sequenceid=162, filesize=30.4 K 2024-12-04T06:43:33,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/86f40af27d314178a5f347f322172489 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489 2024-12-04T06:43:33,553 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489, entries=150, sequenceid=162, filesize=11.9 K 2024-12-04T06:43:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/b3134c394d5f4e3d99107d08c0d79b6c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c 2024-12-04T06:43:33,556 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c, entries=150, sequenceid=162, filesize=11.9 K 2024-12-04T06:43:33,557 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1256ms, sequenceid=162, compaction requested=true 2024-12-04T06:43:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T06:43:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T06:43:33,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T06:43:33,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-12-04T06:43:33,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4140 sec 2024-12-04T06:43:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:33,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-04T06:43:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:33,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d52097143d984b4cb5858747f8d36ae7_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:33,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742401_1577 (size=14794) 2024-12-04T06:43:33,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294673648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294673650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294673696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294673707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294673707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294673756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294673766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294673967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:33,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294673976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:33,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:34,035 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204d52097143d984b4cb5858747f8d36ae7_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d52097143d984b4cb5858747f8d36ae7_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:34,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/8bd092082a99461692f53a5b52a83c52, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:34,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/8bd092082a99461692f53a5b52a83c52 is 175, key is test_row_0/A:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:34,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742402_1578 (size=39749) 2024-12-04T06:43:34,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294674276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294674287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T06:43:34,297 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:34,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-04T06:43:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:34,300 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:34,300 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:34,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:34,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/8bd092082a99461692f53a5b52a83c52 2024-12-04T06:43:34,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/1d189dfc66cc4cc1863a295c14638224 is 50, key is test_row_0/B:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:34,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742403_1579 (size=12151) 2024-12-04T06:43:34,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:34,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294674716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294674716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294674716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,720 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:34,720 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:34,721 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294674786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294674806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:34,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/1d189dfc66cc4cc1863a295c14638224 2024-12-04T06:43:34,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/50d665aa5c614592983b0bcf5ede0b61 is 50, key is test_row_0/C:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:34,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742404_1580 (size=12151) 2024-12-04T06:43:34,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:34,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:34,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:35,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:35,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:35,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:35,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/50d665aa5c614592983b0bcf5ede0b61 2024-12-04T06:43:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/8bd092082a99461692f53a5b52a83c52 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52 2024-12-04T06:43:35,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52, entries=200, sequenceid=183, filesize=38.8 K 2024-12-04T06:43:35,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/1d189dfc66cc4cc1863a295c14638224 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224 2024-12-04T06:43:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224, entries=150, sequenceid=183, filesize=11.9 K 2024-12-04T06:43:35,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/50d665aa5c614592983b0bcf5ede0b61 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61 2024-12-04T06:43:35,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61, entries=150, sequenceid=183, filesize=11.9 K 2024-12-04T06:43:35,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1654ms, sequenceid=183, compaction requested=true 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:35,276 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-04T06:43:35,276 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:35,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:35,278 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 96611 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-04T06:43:35,278 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/B is initiating minor compaction (all files) 2024-12-04T06:43:35,278 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/B in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,278 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/9a02727d35a34eaba5b8c7d2ae363f69, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=94.3 K 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 265481 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/A is initiating minor compaction (all files) 2024-12-04T06:43:35,279 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/A in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,279 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=259.3 K 2024-12-04T06:43:35,279 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=8 throughput controller=DefaultCompactionThroughputController [maxThroughput=61.54 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52] 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a02727d35a34eaba5b8c7d2ae363f69, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f71197f54a1349e58211eed6dc834845, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:35,279 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting a53fec52e31048ebb4a94ce501becb84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733294602812 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 867869329c904325bfc3304e3adca55d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733294602812 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting fb08a430d4714a6081341559a089af59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733294604696 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6521fb0c8d543a5a624375323821713, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733294604696 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting c16236a282e146b9bbc05201f726e986, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733294605387 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4081da702f540459d73f4f11393aa99, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733294605387 2024-12-04T06:43:35,280 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5826536f39ff4069a2da2e794fe6cff6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733294606999 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d7cdfc3358466880128d95584fe5bc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733294606999 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5e423248fd4c7b9e2c99b1e47ee367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733294608171 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf9995c630d40489dd86984ed7f73b7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733294608171 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f40af27d314178a5f347f322172489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733294610339 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d39fbe462a41fda3a6482371155545, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733294610339 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd092082a99461692f53a5b52a83c52, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612546 2024-12-04T06:43:35,281 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d189dfc66cc4cc1863a295c14638224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612556 2024-12-04T06:43:35,294 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#B#compaction#494 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:35,294 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3387a5bd4db9402785e7244e92c3a6a2 is 50, key is test_row_0/B:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:35,295 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:35,301 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241204938ef3b2e86a4231afb651ecc6a91c51_1aa8c1f4b46ff15f781ef8f5e098cbaa store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742405_1581 (size=12527) 2024-12-04T06:43:35,305 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241204938ef3b2e86a4231afb651ecc6a91c51_1aa8c1f4b46ff15f781ef8f5e098cbaa, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:35,305 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204938ef3b2e86a4231afb651ecc6a91c51_1aa8c1f4b46ff15f781ef8f5e098cbaa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742406_1582 (size=4469) 2024-12-04T06:43:35,318 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#A#compaction#495 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:35,319 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4cfb0c6e8e99403ab0005950793ca7f4 is 175, key is test_row_0/A:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742407_1583 (size=31481) 2024-12-04T06:43:35,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e5290c20e85c4794bb6d802cbfb801ec_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294613635/Put/seqid=0 2024-12-04T06:43:35,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742408_1584 (size=12304) 2024-12-04T06:43:35,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:35,385 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204e5290c20e85c4794bb6d802cbfb801ec_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e5290c20e85c4794bb6d802cbfb801ec_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/a6c4bab7ddc7482883f4110385043f79, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/a6c4bab7ddc7482883f4110385043f79 is 175, key is test_row_0/A:col10/1733294613635/Put/seqid=0 2024-12-04T06:43:35,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742409_1585 (size=31105) 2024-12-04T06:43:35,391 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/a6c4bab7ddc7482883f4110385043f79 2024-12-04T06:43:35,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/305e4a0717b142369e8d37a7b83d268f is 50, key is test_row_0/B:col10/1733294613635/Put/seqid=0 2024-12-04T06:43:35,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742410_1586 (size=12151) 2024-12-04T06:43:35,403 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/305e4a0717b142369e8d37a7b83d268f 2024-12-04T06:43:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6f43045c01984d9baf2aacb620e24ced is 50, key is test_row_0/C:col10/1733294613635/Put/seqid=0 2024-12-04T06:43:35,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742411_1587 (size=12151) 2024-12-04T06:43:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:35,709 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3387a5bd4db9402785e7244e92c3a6a2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3387a5bd4db9402785e7244e92c3a6a2 2024-12-04T06:43:35,714 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/B of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 3387a5bd4db9402785e7244e92c3a6a2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:35,714 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:35,714 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/B, priority=8, startTime=1733294615276; duration=0sec 2024-12-04T06:43:35,714 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:35,714 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:B 2024-12-04T06:43:35,714 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-12-04T06:43:35,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 96611 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-12-04T06:43:35,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/C is initiating minor compaction (all files) 2024-12-04T06:43:35,716 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/C in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,716 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/5e0904eedf0d432b8a3d7260547bc6f8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=94.3 K 2024-12-04T06:43:35,716 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0904eedf0d432b8a3d7260547bc6f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733294601631 2024-12-04T06:43:35,717 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a47f6cae214429875bcc763d33cfcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733294602812 2024-12-04T06:43:35,717 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting be95771682624aab92f73248ca946cb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733294604696 2024-12-04T06:43:35,717 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting f667bf2ffa374fd38197c8ade3d80d04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733294605387 2024-12-04T06:43:35,717 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting ed9b549765d74098b14377c68046bbd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733294606999 2024-12-04T06:43:35,717 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 95849e77d26a4963bd644c9192124e6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1733294608171 2024-12-04T06:43:35,718 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting b3134c394d5f4e3d99107d08c0d79b6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733294610339 2024-12-04T06:43:35,718 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d665aa5c614592983b0bcf5ede0b61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612556 2024-12-04T06:43:35,727 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4cfb0c6e8e99403ab0005950793ca7f4 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4 2024-12-04T06:43:35,729 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#C#compaction#499 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:35,730 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/100e28cb3bad4704bb2dd2108ae88693 is 50, key is test_row_0/C:col10/1733294613620/Put/seqid=0 2024-12-04T06:43:35,730 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/A of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 4cfb0c6e8e99403ab0005950793ca7f4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:35,730 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:35,731 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/A, priority=8, startTime=1733294615276; duration=0sec 2024-12-04T06:43:35,731 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:35,731 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:A 2024-12-04T06:43:35,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742412_1588 (size=12527) 2024-12-04T06:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:35,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:35,815 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6f43045c01984d9baf2aacb620e24ced 2024-12-04T06:43:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/a6c4bab7ddc7482883f4110385043f79 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79 2024-12-04T06:43:35,822 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79, entries=150, sequenceid=194, filesize=30.4 K 2024-12-04T06:43:35,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/305e4a0717b142369e8d37a7b83d268f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f 2024-12-04T06:43:35,826 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T06:43:35,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/6f43045c01984d9baf2aacb620e24ced as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced 2024-12-04T06:43:35,830 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced, entries=150, sequenceid=194, filesize=11.9 K 2024-12-04T06:43:35,831 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=46.96 KB/48090 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 461ms, sequenceid=194, compaction requested=false 2024-12-04T06:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-04T06:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:35,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:35,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-04T06:43:35,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-12-04T06:43:35,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5370 sec 2024-12-04T06:43:35,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204934239af08d24a5caaebabf7d461759d_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:35,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742413_1589 (size=14794) 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294615860, max timestamp=1733294615891Num uniques 859; 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294615860, max timestamp=1733294615891Num uniques 859; 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733294615860, max timestamp=1733294615891Num uniques 859; 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-04T06:43:35,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294675942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294675949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294676056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294676066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,142 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/100e28cb3bad4704bb2dd2108ae88693 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/100e28cb3bad4704bb2dd2108ae88693 2024-12-04T06:43:36,146 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/C of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 100e28cb3bad4704bb2dd2108ae88693(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:36,146 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:36,146 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/C, priority=8, startTime=1733294615276; duration=0sec 2024-12-04T06:43:36,146 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:36,146 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:C 2024-12-04T06:43:36,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:36,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204934239af08d24a5caaebabf7d461759d_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204934239af08d24a5caaebabf7d461759d_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:36,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/950357ade02c4f67b3dc8ecd4106731b, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:36,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/950357ade02c4f67b3dc8ecd4106731b is 175, key is test_row_0/A:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742414_1590 (size=39749) 2024-12-04T06:43:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294676267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294676276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T06:43:36,447 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:36,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-04T06:43:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:36,449 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:36,450 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:36,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:36,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294676577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294676587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T06:43:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:36,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/950357ade02c4f67b3dc8ecd4106731b 2024-12-04T06:43:36,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/62348aca64f04f56b6da64218c620d5b is 50, key is test_row_0/B:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:36,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742415_1591 (size=12151) 2024-12-04T06:43:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38720 deadline: 1733294676736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,743 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733294676747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38760 deadline: 1733294676747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:36,753 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_0' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:36,753 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 607fd5c6574c,38257,1733294430862 for 'test_row_1' in 1aa8c1f4b46ff15f781ef8f5e098cbaa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:36,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T06:43:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:36,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T06:43:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:36,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:37,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/62348aca64f04f56b6da64218c620d5b 2024-12-04T06:43:37,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T06:43:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T06:43:37,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b08791fd48c4d05a280fa4662fbc04c is 50, key is test_row_0/C:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:37,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294677087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:37,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:37,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742416_1592 (size=12151) 2024-12-04T06:43:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294677099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:37,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b08791fd48c4d05a280fa4662fbc04c 2024-12-04T06:43:37,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/950357ade02c4f67b3dc8ecd4106731b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b 2024-12-04T06:43:37,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b, entries=200, sequenceid=207, filesize=38.8 K 2024-12-04T06:43:37,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/62348aca64f04f56b6da64218c620d5b as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b 2024-12-04T06:43:37,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T06:43:37,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b08791fd48c4d05a280fa4662fbc04c as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c 2024-12-04T06:43:37,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c, entries=150, sequenceid=207, filesize=11.9 K 2024-12-04T06:43:37,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1312ms, sequenceid=207, compaction requested=true 2024-12-04T06:43:37,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:37,144 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:37,145 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:37,145 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/A is initiating minor compaction (all files) 2024-12-04T06:43:37,145 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/A in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,145 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=99.9 K 2024-12-04T06:43:37,145 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=61.54 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,145 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. files: [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b] 2024-12-04T06:43:37,147 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfb0c6e8e99403ab0005950793ca7f4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612556 2024-12-04T06:43:37,147 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c4bab7ddc7482883f4110385043f79, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294613635 2024-12-04T06:43:37,147 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950357ade02c4f67b3dc8ecd4106731b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294615811 2024-12-04T06:43:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:A, priority=-2147483648, current under compaction store size is 1 2024-12-04T06:43:37,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:37,148 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:37,149 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:37,149 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/B is initiating minor compaction (all files) 2024-12-04T06:43:37,149 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/B in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,150 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3387a5bd4db9402785e7244e92c3a6a2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=36.0 K 2024-12-04T06:43:37,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:B, priority=-2147483648, current under compaction store size is 2 2024-12-04T06:43:37,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:37,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1aa8c1f4b46ff15f781ef8f5e098cbaa:C, priority=-2147483648, current under compaction store size is 3 2024-12-04T06:43:37,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:37,152 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 3387a5bd4db9402785e7244e92c3a6a2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612556 2024-12-04T06:43:37,153 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 305e4a0717b142369e8d37a7b83d268f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294613635 2024-12-04T06:43:37,154 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 62348aca64f04f56b6da64218c620d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294615811 2024-12-04T06:43:37,173 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:37,177 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:37,178 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3acaa95e4fe24e4da56442370f10cbda is 50, key is test_row_0/B:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:37,185 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412041dc70bdf75704c168346fb6f77e58457_1aa8c1f4b46ff15f781ef8f5e098cbaa store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:37,188 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412041dc70bdf75704c168346fb6f77e58457_1aa8c1f4b46ff15f781ef8f5e098cbaa, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:37,188 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412041dc70bdf75704c168346fb6f77e58457_1aa8c1f4b46ff15f781ef8f5e098cbaa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:37,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742417_1593 (size=12629) 2024-12-04T06:43:37,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,218 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:37,221 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/3acaa95e4fe24e4da56442370f10cbda as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3acaa95e4fe24e4da56442370f10cbda 2024-12-04T06:43:37,235 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/B of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 3acaa95e4fe24e4da56442370f10cbda(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:37,235 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:37,235 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/B, priority=13, startTime=1733294617148; duration=0sec 2024-12-04T06:43:37,235 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T06:43:37,235 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:B 2024-12-04T06:43:37,235 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T06:43:37,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742418_1594 (size=4469) 2024-12-04T06:43:37,237 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T06:43:37,237 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#A#compaction#503 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:37,237 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1541): 1aa8c1f4b46ff15f781ef8f5e098cbaa/C is initiating minor compaction (all files) 2024-12-04T06:43:37,238 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1aa8c1f4b46ff15f781ef8f5e098cbaa/C in TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:37,238 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/100e28cb3bad4704bb2dd2108ae88693, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c] into tmpdir=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp, totalSize=36.0 K 2024-12-04T06:43:37,238 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/9b528190175a4d97bc00b2cefc440e02 is 175, key is test_row_0/A:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:37,238 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 100e28cb3bad4704bb2dd2108ae88693, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733294612556 2024-12-04T06:43:37,240 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f43045c01984d9baf2aacb620e24ced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733294613635 2024-12-04T06:43:37,241 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b08791fd48c4d05a280fa4662fbc04c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733294615811 2024-12-04T06:43:37,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742419_1595 (size=31583) 2024-12-04T06:43:37,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b248c2c4cb8e4ef290db45af671e6aa9_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294615941/Put/seqid=0 2024-12-04T06:43:37,249 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1aa8c1f4b46ff15f781ef8f5e098cbaa#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 61.54 MB/second 2024-12-04T06:43:37,250 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/a4b345f2ac124a10bf98a3728ae7dc9a is 50, key is test_row_0/C:col10/1733294615825/Put/seqid=0 2024-12-04T06:43:37,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742420_1596 (size=12304) 2024-12-04T06:43:37,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:37,257 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204b248c2c4cb8e4ef290db45af671e6aa9_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b248c2c4cb8e4ef290db45af671e6aa9_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3a466dc56e6b4b44a210f9f530aae9fe, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3a466dc56e6b4b44a210f9f530aae9fe is 175, key is test_row_0/A:col10/1733294615941/Put/seqid=0 2024-12-04T06:43:37,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742421_1597 (size=12629) 2024-12-04T06:43:37,285 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/a4b345f2ac124a10bf98a3728ae7dc9a as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/a4b345f2ac124a10bf98a3728ae7dc9a 2024-12-04T06:43:37,290 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/C of 1aa8c1f4b46ff15f781ef8f5e098cbaa into a4b345f2ac124a10bf98a3728ae7dc9a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:37,290 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:37,290 INFO [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/C, priority=13, startTime=1733294617151; duration=0sec 2024-12-04T06:43:37,290 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:37,290 DEBUG [RS:0;607fd5c6574c:38257-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:C 2024-12-04T06:43:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742422_1598 (size=31105) 2024-12-04T06:43:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:37,650 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/9b528190175a4d97bc00b2cefc440e02 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/9b528190175a4d97bc00b2cefc440e02 2024-12-04T06:43:37,654 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1aa8c1f4b46ff15f781ef8f5e098cbaa/A of 1aa8c1f4b46ff15f781ef8f5e098cbaa into 9b528190175a4d97bc00b2cefc440e02(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T06:43:37,654 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:37,654 INFO [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., storeName=1aa8c1f4b46ff15f781ef8f5e098cbaa/A, priority=13, startTime=1733294617144; duration=0sec 2024-12-04T06:43:37,654 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T06:43:37,654 DEBUG [RS:0;607fd5c6574c:38257-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1aa8c1f4b46ff15f781ef8f5e098cbaa:A 2024-12-04T06:43:37,691 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3a466dc56e6b4b44a210f9f530aae9fe 2024-12-04T06:43:37,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/50fdfa2be49648209118d202d74536bc is 50, key is test_row_0/B:col10/1733294615941/Put/seqid=0 2024-12-04T06:43:37,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742423_1599 (size=12151) 2024-12-04T06:43:38,103 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/50fdfa2be49648209118d202d74536bc 2024-12-04T06:43:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:38,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b06844bf8a24de2bfbee11d96fd94ee is 50, key is test_row_0/C:col10/1733294615941/Put/seqid=0 2024-12-04T06:43:38,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742424_1600 (size=12151) 2024-12-04T06:43:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294678131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294678131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294678247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294678247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294678456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,461 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294678456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,517 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b06844bf8a24de2bfbee11d96fd94ee 2024-12-04T06:43:38,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/3a466dc56e6b4b44a210f9f530aae9fe as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3a466dc56e6b4b44a210f9f530aae9fe 2024-12-04T06:43:38,524 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3a466dc56e6b4b44a210f9f530aae9fe, entries=150, sequenceid=243, filesize=30.4 K 2024-12-04T06:43:38,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/50fdfa2be49648209118d202d74536bc as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/50fdfa2be49648209118d202d74536bc 2024-12-04T06:43:38,528 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/50fdfa2be49648209118d202d74536bc, entries=150, sequenceid=243, filesize=11.9 K 2024-12-04T06:43:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/8b06844bf8a24de2bfbee11d96fd94ee as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b06844bf8a24de2bfbee11d96fd94ee 2024-12-04T06:43:38,531 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b06844bf8a24de2bfbee11d96fd94ee, entries=150, sequenceid=243, filesize=11.9 K 2024-12-04T06:43:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1314ms, sequenceid=243, compaction requested=false 2024-12-04T06:43:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-04T06:43:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-04T06:43:38,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-04T06:43:38,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0830 sec 2024-12-04T06:43:38,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0880 sec 2024-12-04T06:43:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-04T06:43:38,597 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:38,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-04T06:43:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T06:43:38,599 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:38,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:38,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T06:43:38,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-04T06:43:38,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:38,752 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:38,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465658f3fc56541b6a689f933e1c5bd10_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294618124/Put/seqid=0 2024-12-04T06:43:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742425_1601 (size=12304) 2024-12-04T06:43:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:38,766 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120465658f3fc56541b6a689f933e1c5bd10_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465658f3fc56541b6a689f933e1c5bd10_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4890ac2482bc4c0e9986ee398c62a138, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4890ac2482bc4c0e9986ee398c62a138 is 175, key is test_row_0/A:col10/1733294618124/Put/seqid=0 2024-12-04T06:43:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:38,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742426_1602 (size=31105) 2024-12-04T06:43:38,787 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4890ac2482bc4c0e9986ee398c62a138 2024-12-04T06:43:38,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/067ffc7998b7439fad52e6cae52d6c66 is 50, key is test_row_0/B:col10/1733294618124/Put/seqid=0 2024-12-04T06:43:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742427_1603 (size=12151) 2024-12-04T06:43:38,797 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/067ffc7998b7439fad52e6cae52d6c66 2024-12-04T06:43:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/68cbfb9886bb468bbccd667e8055c4ef is 50, key is test_row_0/C:col10/1733294618124/Put/seqid=0 2024-12-04T06:43:38,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742428_1604 (size=12151) 2024-12-04T06:43:38,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294618807, max timestamp=1733294618819Num uniques 859; 2024-12-04T06:43:38,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:43:38,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294618807, max timestamp=1733294618819Num uniques 859; 2024-12-04T06:43:38,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:43:38,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733294618807, max timestamp=1733294618819Num uniques 859; 2024-12-04T06:43:38,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/607fd5c6574c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-04T06:43:38,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294678867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294678868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T06:43:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294678977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294678987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294679186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294679197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,206 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/68cbfb9886bb468bbccd667e8055c4ef 2024-12-04T06:43:39,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/4890ac2482bc4c0e9986ee398c62a138 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4890ac2482bc4c0e9986ee398c62a138 2024-12-04T06:43:39,213 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4890ac2482bc4c0e9986ee398c62a138, entries=150, sequenceid=257, filesize=30.4 K 2024-12-04T06:43:39,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/067ffc7998b7439fad52e6cae52d6c66 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/067ffc7998b7439fad52e6cae52d6c66 2024-12-04T06:43:39,217 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/067ffc7998b7439fad52e6cae52d6c66, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T06:43:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/68cbfb9886bb468bbccd667e8055c4ef as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/68cbfb9886bb468bbccd667e8055c4ef 2024-12-04T06:43:39,220 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/68cbfb9886bb468bbccd667e8055c4ef, entries=150, sequenceid=257, filesize=11.9 K 2024-12-04T06:43:39,221 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 469ms, sequenceid=257, compaction requested=true 2024-12-04T06:43:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-04T06:43:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-04T06:43:39,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-04T06:43:39,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2024-12-04T06:43:39,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 628 msec 2024-12-04T06:43:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T06:43:39,227 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:39,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-04T06:43:39,229 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T06:43:39,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:39,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T06:43:39,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:39,382 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-04T06:43:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049eb5b0fcb0a24b158b1a2c9b59611dcc_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294618852/Put/seqid=0 2024-12-04T06:43:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742429_1605 (size=12454) 2024-12-04T06:43:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:39,396 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412049eb5b0fcb0a24b158b1a2c9b59611dcc_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049eb5b0fcb0a24b158b1a2c9b59611dcc_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/55f7d32e5e8d408dbaa4736ec06e022e, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/55f7d32e5e8d408dbaa4736ec06e022e is 175, key is test_row_0/A:col10/1733294618852/Put/seqid=0 2024-12-04T06:43:39,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742430_1606 (size=31255) 2024-12-04T06:43:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:39,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294679520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294679523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T06:43:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294679636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294679637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,802 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/55f7d32e5e8d408dbaa4736ec06e022e 2024-12-04T06:43:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/2ed2e2bc2ec5403ea37c85a9a8e29bed is 50, key is test_row_0/B:col10/1733294618852/Put/seqid=0 2024-12-04T06:43:39,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742431_1607 (size=12301) 2024-12-04T06:43:39,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294679846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294679846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T06:43:40,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1733294680157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:40,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:40,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38257 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1733294680157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 , the old value is region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T06:43:40,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., hostname=607fd5c6574c,38257,1733294430862, seqNum=5 because the exception is null or not the one we care about 2024-12-04T06:43:40,212 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/2ed2e2bc2ec5403ea37c85a9a8e29bed 2024-12-04T06:43:40,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/1f801780d4f84794940254e7b05a8473 is 50, key is test_row_0/C:col10/1733294618852/Put/seqid=0 2024-12-04T06:43:40,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742432_1608 (size=12301) 2024-12-04T06:43:40,226 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/1f801780d4f84794940254e7b05a8473 2024-12-04T06:43:40,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/55f7d32e5e8d408dbaa4736ec06e022e as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/55f7d32e5e8d408dbaa4736ec06e022e 2024-12-04T06:43:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/55f7d32e5e8d408dbaa4736ec06e022e, entries=150, sequenceid=293, filesize=30.5 K 2024-12-04T06:43:40,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/2ed2e2bc2ec5403ea37c85a9a8e29bed as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/2ed2e2bc2ec5403ea37c85a9a8e29bed 2024-12-04T06:43:40,238 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/2ed2e2bc2ec5403ea37c85a9a8e29bed, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T06:43:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/1f801780d4f84794940254e7b05a8473 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/1f801780d4f84794940254e7b05a8473 2024-12-04T06:43:40,242 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/1f801780d4f84794940254e7b05a8473, entries=150, sequenceid=293, filesize=12.0 K 2024-12-04T06:43:40,243 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 861ms, sequenceid=293, compaction requested=true 2024-12-04T06:43:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-04T06:43:40,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-04T06:43:40,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-04T06:43:40,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-12-04T06:43:40,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0200 sec 2024-12-04T06:43:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-04T06:43:40,367 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:40,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-04T06:43:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-04T06:43:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:40,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-04T06:43:40,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-04T06:43:40,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-04T06:43:40,387 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-12-04T06:43:40,387 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,387 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,387 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,388 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-12-04T06:43:40,388 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,388 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,388 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,390 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-12-04T06:43:40,390 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-12-04T06:43:40,390 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,390 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,390 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,390 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,390 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,390 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,391 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-12-04T06:43:40,391 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,391 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,391 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:40,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38257 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-04T06:43:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:40,522 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-04T06:43:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:40,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bb1f1ed1fb954adc99952485515c73d0_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294619521/Put/seqid=0 2024-12-04T06:43:40,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742433_1609 (size=12454) 2024-12-04T06:43:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38257 {}] regionserver.HRegion(8855): Flush requested on 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:40,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. as already flushing 2024-12-04T06:43:40,668 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-04T06:43:40,668 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,668 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,668 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,678 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-12-04T06:43:40,678 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,678 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,678 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:40,778 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-04T06:43:40,778 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-12-04T06:43:40,778 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,778 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,778 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,778 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,779 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,779 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,787 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-12-04T06:43:40,787 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-04T06:43:40,787 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,787 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:40,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:40,934 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204bb1f1ed1fb954adc99952485515c73d0_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bb1f1ed1fb954adc99952485515c73d0_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:40,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ae3456aa6d6a469f969c6ca3c37d5fc8, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:40,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ae3456aa6d6a469f969c6ca3c37d5fc8 is 175, key is test_row_0/A:col10/1733294619521/Put/seqid=0 2024-12-04T06:43:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742434_1610 (size=31255) 2024-12-04T06:43:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:41,339 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ae3456aa6d6a469f969c6ca3c37d5fc8 2024-12-04T06:43:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/82ab14af428448c2ab3894c2da52e6f2 is 50, key is test_row_0/B:col10/1733294619521/Put/seqid=0 2024-12-04T06:43:41,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742435_1611 (size=12301) 2024-12-04T06:43:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:41,747 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/82ab14af428448c2ab3894c2da52e6f2 2024-12-04T06:43:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/46ce4f5130884f86a3d5e37023f75b96 is 50, key is test_row_0/C:col10/1733294619521/Put/seqid=0 2024-12-04T06:43:41,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742436_1612 (size=12301) 2024-12-04T06:43:42,156 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/46ce4f5130884f86a3d5e37023f75b96 2024-12-04T06:43:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/ae3456aa6d6a469f969c6ca3c37d5fc8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ae3456aa6d6a469f969c6ca3c37d5fc8 2024-12-04T06:43:42,162 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ae3456aa6d6a469f969c6ca3c37d5fc8, entries=150, sequenceid=304, filesize=30.5 K 2024-12-04T06:43:42,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/82ab14af428448c2ab3894c2da52e6f2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/82ab14af428448c2ab3894c2da52e6f2 2024-12-04T06:43:42,165 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/82ab14af428448c2ab3894c2da52e6f2, entries=150, sequenceid=304, filesize=12.0 K 2024-12-04T06:43:42,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/46ce4f5130884f86a3d5e37023f75b96 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/46ce4f5130884f86a3d5e37023f75b96 2024-12-04T06:43:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/46ce4f5130884f86a3d5e37023f75b96, entries=150, sequenceid=304, filesize=12.0 K 2024-12-04T06:43:42,169 INFO [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=33.54 KB/34350 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1647ms, sequenceid=304, compaction requested=true 2024-12-04T06:43:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: 2024-12-04T06:43:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/607fd5c6574c:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-04T06:43:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-04T06:43:42,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-04T06:43:42,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-04T06:43:42,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8040 sec 2024-12-04T06:43:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T06:43:42,517 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2582 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2507 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2532 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2594 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2501 2024-12-04T06:43:42,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-04T06:43:42,517 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:43:42,517 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:43:42,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:42,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:42,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:42,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-04T06:43:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:42,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294622520"}]},"ts":"1733294622520"} 2024-12-04T06:43:42,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-04T06:43:42,522 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-04T06:43:42,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-04T06:43:42,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, UNASSIGN}] 2024-12-04T06:43:42,524 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, UNASSIGN 2024-12-04T06:43:42,524 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=CLOSING, regionLocation=607fd5c6574c,38257,1733294430862 2024-12-04T06:43:42,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, UNASSIGN because future has completed 2024-12-04T06:43:42,526 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T06:43:42,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862}] 2024-12-04T06:43:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:42,678 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 1aa8c1f4b46ff15f781ef8f5e098cbaa, disabling compactions & flushes 2024-12-04T06:43:42,678 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. after waiting 0 ms 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:42,678 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=A 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=B 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1aa8c1f4b46ff15f781ef8f5e098cbaa, store=C 2024-12-04T06:43:42,678 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-04T06:43:42,682 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120401c2552631a94552803b82cbd3ad0022_1aa8c1f4b46ff15f781ef8f5e098cbaa is 50, key is test_row_0/A:col10/1733294620786/Put/seqid=0 2024-12-04T06:43:42,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742437_1613 (size=9914) 2024-12-04T06:43:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:43,086 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T06:43:43,089 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120401c2552631a94552803b82cbd3ad0022_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120401c2552631a94552803b82cbd3ad0022_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:43,089 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/1e7dd5bf46c041fdb770ab0bb750a905, store: [table=TestAcidGuarantees family=A region=1aa8c1f4b46ff15f781ef8f5e098cbaa] 2024-12-04T06:43:43,090 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/1e7dd5bf46c041fdb770ab0bb750a905 is 175, key is test_row_0/A:col10/1733294620786/Put/seqid=0 2024-12-04T06:43:43,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742438_1614 (size=22561) 2024-12-04T06:43:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:43,494 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/1e7dd5bf46c041fdb770ab0bb750a905 2024-12-04T06:43:43,499 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/f55ca70539d44b38890eb3fa72ec8bf0 is 50, key is test_row_0/B:col10/1733294620786/Put/seqid=0 2024-12-04T06:43:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742439_1615 (size=9857) 2024-12-04T06:43:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:43,902 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/f55ca70539d44b38890eb3fa72ec8bf0 2024-12-04T06:43:43,908 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/4a723fb59edb4d04809feb5df5c64eb8 is 50, key is test_row_0/C:col10/1733294620786/Put/seqid=0 2024-12-04T06:43:43,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742440_1616 (size=9857) 2024-12-04T06:43:44,311 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/4a723fb59edb4d04809feb5df5c64eb8 2024-12-04T06:43:44,314 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/A/1e7dd5bf46c041fdb770ab0bb750a905 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/1e7dd5bf46c041fdb770ab0bb750a905 2024-12-04T06:43:44,317 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/1e7dd5bf46c041fdb770ab0bb750a905, entries=100, sequenceid=312, filesize=22.0 K 2024-12-04T06:43:44,317 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/B/f55ca70539d44b38890eb3fa72ec8bf0 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/f55ca70539d44b38890eb3fa72ec8bf0 2024-12-04T06:43:44,320 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/f55ca70539d44b38890eb3fa72ec8bf0, entries=100, sequenceid=312, filesize=9.6 K 2024-12-04T06:43:44,320 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/.tmp/C/4a723fb59edb4d04809feb5df5c64eb8 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/4a723fb59edb4d04809feb5df5c64eb8 2024-12-04T06:43:44,323 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/4a723fb59edb4d04809feb5df5c64eb8, entries=100, sequenceid=312, filesize=9.6 K 2024-12-04T06:43:44,323 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1645ms, sequenceid=312, compaction requested=true 2024-12-04T06:43:44,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b] to archive 2024-12-04T06:43:44,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:44,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/5c0203c4774d440f8bfb32b7bc699e19 2024-12-04T06:43:44,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ba909a5870d3456b864dee1467f310a3 2024-12-04T06:43:44,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/2d9c0ddf899046e0bd004e961041113e 2024-12-04T06:43:44,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f71197f54a1349e58211eed6dc834845 2024-12-04T06:43:44,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/867869329c904325bfc3304e3adca55d 2024-12-04T06:43:44,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/b6521fb0c8d543a5a624375323821713 2024-12-04T06:43:44,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/f4081da702f540459d73f4f11393aa99 2024-12-04T06:43:44,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/c3d7cdfc3358466880128d95584fe5bc 2024-12-04T06:43:44,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3bf9995c630d40489dd86984ed7f73b7 2024-12-04T06:43:44,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/04d39fbe462a41fda3a6482371155545 2024-12-04T06:43:44,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/8bd092082a99461692f53a5b52a83c52 2024-12-04T06:43:44,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4cfb0c6e8e99403ab0005950793ca7f4 2024-12-04T06:43:44,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/a6c4bab7ddc7482883f4110385043f79 2024-12-04T06:43:44,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/950357ade02c4f67b3dc8ecd4106731b 2024-12-04T06:43:44,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/9a02727d35a34eaba5b8c7d2ae363f69, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3387a5bd4db9402785e7244e92c3a6a2, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b] to archive 2024-12-04T06:43:44,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:44,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/b6981160b0bf4718b9f9f1ace55d373c 2024-12-04T06:43:44,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/0b6092f0c56541d8b52df26f6e147f28 2024-12-04T06:43:44,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/9a02727d35a34eaba5b8c7d2ae363f69 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/9a02727d35a34eaba5b8c7d2ae363f69 2024-12-04T06:43:44,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3bcd37d27f7d44388438fd9943a71222 2024-12-04T06:43:44,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/a53fec52e31048ebb4a94ce501becb84 2024-12-04T06:43:44,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/fb08a430d4714a6081341559a089af59 2024-12-04T06:43:44,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/c16236a282e146b9bbc05201f726e986 2024-12-04T06:43:44,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/5826536f39ff4069a2da2e794fe6cff6 2024-12-04T06:43:44,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3c5e423248fd4c7b9e2c99b1e47ee367 2024-12-04T06:43:44,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/86f40af27d314178a5f347f322172489 2024-12-04T06:43:44,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3387a5bd4db9402785e7244e92c3a6a2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3387a5bd4db9402785e7244e92c3a6a2 2024-12-04T06:43:44,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/1d189dfc66cc4cc1863a295c14638224 2024-12-04T06:43:44,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/305e4a0717b142369e8d37a7b83d268f 2024-12-04T06:43:44,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/62348aca64f04f56b6da64218c620d5b 2024-12-04T06:43:44,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/5e0904eedf0d432b8a3d7260547bc6f8, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/100e28cb3bad4704bb2dd2108ae88693, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c] to archive 2024-12-04T06:43:44,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T06:43:44,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/c22fb14066a8449ab1cbd973df12795a 2024-12-04T06:43:44,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6eaeec6e2beb4257bd33f4726e3cd696 2024-12-04T06:43:44,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/5e0904eedf0d432b8a3d7260547bc6f8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/5e0904eedf0d432b8a3d7260547bc6f8 2024-12-04T06:43:44,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/25014f33e76d4500843138871fb3d273 2024-12-04T06:43:44,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/d2a47f6cae214429875bcc763d33cfcc 2024-12-04T06:43:44,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/be95771682624aab92f73248ca946cb3 2024-12-04T06:43:44,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/f667bf2ffa374fd38197c8ade3d80d04 2024-12-04T06:43:44,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/ed9b549765d74098b14377c68046bbd3 2024-12-04T06:43:44,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/95849e77d26a4963bd644c9192124e6c 2024-12-04T06:43:44,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/b3134c394d5f4e3d99107d08c0d79b6c 2024-12-04T06:43:44,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/100e28cb3bad4704bb2dd2108ae88693 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/100e28cb3bad4704bb2dd2108ae88693 2024-12-04T06:43:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/50d665aa5c614592983b0bcf5ede0b61 2024-12-04T06:43:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/6f43045c01984d9baf2aacb620e24ced 2024-12-04T06:43:44,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b08791fd48c4d05a280fa4662fbc04c 2024-12-04T06:43:44,364 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits/315.seqid, newMaxSeqId=315, maxSeqId=4 2024-12-04T06:43:44,364 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. 2024-12-04T06:43:44,365 DEBUG [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 1aa8c1f4b46ff15f781ef8f5e098cbaa: Waiting for close lock at 1733294622678Running coprocessor pre-close hooks at 1733294622678Disabling compacts and flushes for region at 1733294622678Disabling writes for close at 1733294622678Obtaining lock to block concurrent updates at 1733294622678Preparing flush snapshotting stores in 1aa8c1f4b46ff15f781ef8f5e098cbaa at 1733294622678Finished memstore snapshotting TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733294622678Flushing stores of TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa. at 1733294622679 (+1 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/A: creating writer at 1733294622679Flushing mob file 1aa8c1f4b46ff15f781ef8f5e098cbaa/A: appending metadata at 1733294622682 (+3 ms)Flushing mob file 1aa8c1f4b46ff15f781ef8f5e098cbaa/A: closing flushed file at 1733294622682Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/A: appending metadata at 1733294623090 (+408 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/A: closing flushed file at 1733294623090Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/B: creating writer at 1733294623497 (+407 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/B: appending metadata at 1733294623498 (+1 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/B: closing flushed file at 1733294623499 (+1 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/C: creating writer at 1733294623906 (+407 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/C: appending metadata at 1733294623907 (+1 ms)Flushing 1aa8c1f4b46ff15f781ef8f5e098cbaa/C: closing flushed file at 1733294623907Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27ebecaa: reopening flushed file at 1733294624314 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c43f987: reopening flushed file at 1733294624317 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69662e06: reopening flushed file at 1733294624320 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1aa8c1f4b46ff15f781ef8f5e098cbaa in 1645ms, sequenceid=312, compaction requested=true at 1733294624323 (+3 ms)Writing region close event to WAL at 1733294624362 (+39 ms)Running coprocessor post-close hooks at 1733294624364 (+2 ms)Closed at 1733294624364 2024-12-04T06:43:44,366 INFO [RS_CLOSE_REGION-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=1aa8c1f4b46ff15f781ef8f5e098cbaa, regionState=CLOSED 2024-12-04T06:43:44,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 because future has completed 2024-12-04T06:43:44,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-04T06:43:44,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 1aa8c1f4b46ff15f781ef8f5e098cbaa, server=607fd5c6574c,38257,1733294430862 in 1.8420 sec 2024-12-04T06:43:44,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-04T06:43:44,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1aa8c1f4b46ff15f781ef8f5e098cbaa, UNASSIGN in 1.8460 sec 2024-12-04T06:43:44,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-04T06:43:44,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8490 sec 2024-12-04T06:43:44,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733294624373"}]},"ts":"1733294624373"} 2024-12-04T06:43:44,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-04T06:43:44,374 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-04T06:43:44,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-04T06:43:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-04T06:43:44,667 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:44,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-04T06:43:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-04T06:43:44,670 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,674 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C, FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits] 2024-12-04T06:43:44,675 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/1e7dd5bf46c041fdb770ab0bb750a905 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/1e7dd5bf46c041fdb770ab0bb750a905 2024-12-04T06:43:44,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3a466dc56e6b4b44a210f9f530aae9fe to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/3a466dc56e6b4b44a210f9f530aae9fe 2024-12-04T06:43:44,677 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4890ac2482bc4c0e9986ee398c62a138 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/4890ac2482bc4c0e9986ee398c62a138 2024-12-04T06:43:44,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/55f7d32e5e8d408dbaa4736ec06e022e to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/55f7d32e5e8d408dbaa4736ec06e022e 2024-12-04T06:43:44,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/9b528190175a4d97bc00b2cefc440e02 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/9b528190175a4d97bc00b2cefc440e02 2024-12-04T06:43:44,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ae3456aa6d6a469f969c6ca3c37d5fc8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/A/ae3456aa6d6a469f969c6ca3c37d5fc8 2024-12-04T06:43:44,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/067ffc7998b7439fad52e6cae52d6c66 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/067ffc7998b7439fad52e6cae52d6c66 2024-12-04T06:43:44,682 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/2ed2e2bc2ec5403ea37c85a9a8e29bed to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/2ed2e2bc2ec5403ea37c85a9a8e29bed 2024-12-04T06:43:44,682 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3acaa95e4fe24e4da56442370f10cbda to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/3acaa95e4fe24e4da56442370f10cbda 2024-12-04T06:43:44,683 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/50fdfa2be49648209118d202d74536bc to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/50fdfa2be49648209118d202d74536bc 2024-12-04T06:43:44,684 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/82ab14af428448c2ab3894c2da52e6f2 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/82ab14af428448c2ab3894c2da52e6f2 2024-12-04T06:43:44,685 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/f55ca70539d44b38890eb3fa72ec8bf0 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/B/f55ca70539d44b38890eb3fa72ec8bf0 2024-12-04T06:43:44,686 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/1f801780d4f84794940254e7b05a8473 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/1f801780d4f84794940254e7b05a8473 2024-12-04T06:43:44,687 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/46ce4f5130884f86a3d5e37023f75b96 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/46ce4f5130884f86a3d5e37023f75b96 2024-12-04T06:43:44,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/4a723fb59edb4d04809feb5df5c64eb8 to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/4a723fb59edb4d04809feb5df5c64eb8 2024-12-04T06:43:44,688 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/68cbfb9886bb468bbccd667e8055c4ef to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/68cbfb9886bb468bbccd667e8055c4ef 2024-12-04T06:43:44,689 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b06844bf8a24de2bfbee11d96fd94ee to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/8b06844bf8a24de2bfbee11d96fd94ee 2024-12-04T06:43:44,690 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/a4b345f2ac124a10bf98a3728ae7dc9a to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/C/a4b345f2ac124a10bf98a3728ae7dc9a 2024-12-04T06:43:44,691 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits/315.seqid to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa/recovered.edits/315.seqid 2024-12-04T06:43:44,692 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/default/TestAcidGuarantees/1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,692 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-04T06:43:44,692 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:43:44,693 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-04T06:43:44,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120401c2552631a94552803b82cbd3ad0022_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120401c2552631a94552803b82cbd3ad0022_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,695 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418603e9de38246e99b1bee6489490798_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120418603e9de38246e99b1bee6489490798_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,696 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120424650512c56e424baab256ad49a57779_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120424650512c56e424baab256ad49a57779_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,697 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d6e75f3354b46658e34566a457bdeb5_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412045d6e75f3354b46658e34566a457bdeb5_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465658f3fc56541b6a689f933e1c5bd10_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465658f3fc56541b6a689f933e1c5bd10_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465891534cfff4afea7d903390305640d_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120465891534cfff4afea7d903390305640d_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120469d12e39074a4d20a839562f217341cf_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120469d12e39074a4d20a839562f217341cf_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,700 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204934239af08d24a5caaebabf7d461759d_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204934239af08d24a5caaebabf7d461759d_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,701 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049eb5b0fcb0a24b158b1a2c9b59611dcc_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412049eb5b0fcb0a24b158b1a2c9b59611dcc_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,701 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204af7215c743a44715a945c209452437eb_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204af7215c743a44715a945c209452437eb_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b057697f390b4c01a2ce24c8a9d6b3d7_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b057697f390b4c01a2ce24c8a9d6b3d7_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b248c2c4cb8e4ef290db45af671e6aa9_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b248c2c4cb8e4ef290db45af671e6aa9_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b31a2ae85a5943f3a26eb64407065b35_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204b31a2ae85a5943f3a26eb64407065b35_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,704 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bb1f1ed1fb954adc99952485515c73d0_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204bb1f1ed1fb954adc99952485515c73d0_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,705 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d52097143d984b4cb5858747f8d36ae7_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204d52097143d984b4cb5858747f8d36ae7_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e155436b1c4a4dff9021310558d84118_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e155436b1c4a4dff9021310558d84118_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e5290c20e85c4794bb6d802cbfb801ec_1aa8c1f4b46ff15f781ef8f5e098cbaa to hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241204e5290c20e85c4794bb6d802cbfb801ec_1aa8c1f4b46ff15f781ef8f5e098cbaa 2024-12-04T06:43:44,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-04T06:43:44,708 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,710 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-04T06:43:44,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-04T06:43:44,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,712 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-04T06:43:44,713 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733294624712"}]},"ts":"9223372036854775807"} 2024-12-04T06:43:44,714 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T06:43:44,714 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1aa8c1f4b46ff15f781ef8f5e098cbaa, NAME => 'TestAcidGuarantees,,1733294597294.1aa8c1f4b46ff15f781ef8f5e098cbaa.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T06:43:44,714 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-04T06:43:44,714 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733294624714"}]},"ts":"9223372036854775807"} 2024-12-04T06:43:44,715 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-04T06:43:44,716 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-04T06:43:44,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-12-04T06:43:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45619 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-04T06:43:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-04T06:43:44,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-04T06:43:44,786 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=5747 (was 5836) 2024-12-04T06:43:44,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T06:43:44,787 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:43:44,787 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:43:44,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:44,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:44,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T06:43:44,787 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T06:43:44,787 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=164229068, stopped=false 2024-12-04T06:43:44,787 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=607fd5c6574c,45619,1733294430018 2024-12-04T06:43:44,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T06:43:44,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T06:43:44,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:43:44,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:43:44,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T06:43:44,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T06:43:44,790 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T06:43:44,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T06:43:44,790 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:43:44,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:44,791 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '607fd5c6574c,38257,1733294430862' ***** 2024-12-04T06:43:44,791 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T06:43:44,791 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T06:43:44,792 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(959): stopping server 607fd5c6574c,38257,1733294430862 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;607fd5c6574c:38257. 2024-12-04T06:43:44,792 DEBUG [RS:0;607fd5c6574c:38257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T06:43:44,792 DEBUG [RS:0;607fd5c6574c:38257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T06:43:44,792 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T06:43:44,793 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T06:43:44,793 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T06:43:44,794 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T06:43:44,794 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:44,794 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T06:43:44,794 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T06:43:44,794 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T06:43:44,794 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T06:43:44,794 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-04T06:43:44,817 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/info/40052eed774240b9838cda674eac1ced is 95, key is TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184./info:/1733294466485/DeleteFamily/seqid=0 2024-12-04T06:43:44,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742441_1617 (size=6514) 2024-12-04T06:43:44,876 INFO [regionserver/607fd5c6574c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T06:43:44,994 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:45,194 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:45,220 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/info/40052eed774240b9838cda674eac1ced 2024-12-04T06:43:45,238 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/ns/083b5432a4da4214b06ffad11ce792bf is 93, key is TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184./ns:/1733294466441/DeleteFamily/seqid=0 2024-12-04T06:43:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742442_1618 (size=5906) 2024-12-04T06:43:45,394 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:45,594 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:45,641 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/ns/083b5432a4da4214b06ffad11ce792bf 2024-12-04T06:43:45,658 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/rep_barrier/97e743e8040f409f9c5f28003d1a318f is 102, key is TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184./rep_barrier:/1733294466441/DeleteFamily/seqid=0 2024-12-04T06:43:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742443_1619 (size=6025) 2024-12-04T06:43:45,708 INFO [regionserver/607fd5c6574c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T06:43:45,708 INFO [regionserver/607fd5c6574c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T06:43:45,794 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:45,995 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:46,062 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/rep_barrier/97e743e8040f409f9c5f28003d1a318f 2024-12-04T06:43:46,079 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/table/1884d75a19a744ce83828a9a32f0fd91 is 96, key is TestAcidGuarantees,,1733294433601.c8a5d67afecdb7afe2ddb08e0bd45184./table:/1733294466441/DeleteFamily/seqid=0 2024-12-04T06:43:46,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742444_1620 (size=5936) 2024-12-04T06:43:46,195 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:46,395 DEBUG [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T06:43:46,482 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/table/1884d75a19a744ce83828a9a32f0fd91 2024-12-04T06:43:46,485 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/info/40052eed774240b9838cda674eac1ced as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/info/40052eed774240b9838cda674eac1ced 2024-12-04T06:43:46,488 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/info/40052eed774240b9838cda674eac1ced, entries=12, sequenceid=90, filesize=6.4 K 2024-12-04T06:43:46,488 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/ns/083b5432a4da4214b06ffad11ce792bf as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/ns/083b5432a4da4214b06ffad11ce792bf 2024-12-04T06:43:46,491 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/ns/083b5432a4da4214b06ffad11ce792bf, entries=8, sequenceid=90, filesize=5.8 K 2024-12-04T06:43:46,491 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/rep_barrier/97e743e8040f409f9c5f28003d1a318f as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/rep_barrier/97e743e8040f409f9c5f28003d1a318f 2024-12-04T06:43:46,493 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/rep_barrier/97e743e8040f409f9c5f28003d1a318f, entries=6, sequenceid=90, filesize=5.9 K 2024-12-04T06:43:46,494 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/.tmp/table/1884d75a19a744ce83828a9a32f0fd91 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/table/1884d75a19a744ce83828a9a32f0fd91 2024-12-04T06:43:46,496 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/table/1884d75a19a744ce83828a9a32f0fd91, entries=7, sequenceid=90, filesize=5.8 K 2024-12-04T06:43:46,497 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-12-04T06:43:46,500 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-04T06:43:46,501 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T06:43:46,501 INFO [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T06:43:46,501 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733294624793Running coprocessor pre-close hooks at 1733294624793Disabling compacts and flushes for region at 1733294624793Disabling writes for close at 1733294624794 (+1 ms)Obtaining lock to block concurrent updates at 1733294624794Preparing flush snapshotting stores in 1588230740 at 1733294624794Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733294624794Flushing stores of hbase:meta,,1.1588230740 at 1733294624795 (+1 ms)Flushing 1588230740/info: creating writer at 1733294624795Flushing 1588230740/info: appending metadata at 1733294624816 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733294624816Flushing 1588230740/ns: creating writer at 1733294625224 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733294625237 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733294625237Flushing 1588230740/rep_barrier: creating writer at 1733294625644 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733294625658 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733294625658Flushing 1588230740/table: creating writer at 1733294626064 (+406 ms)Flushing 1588230740/table: appending metadata at 1733294626078 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733294626078Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69bd2976: reopening flushed file at 1733294626485 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e79595a: reopening flushed file at 1733294626488 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@282d6473: reopening flushed file at 1733294626491 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d129603: reopening flushed file at 1733294626493 (+2 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1733294626497 (+4 ms)Writing region close event to WAL at 1733294626498 (+1 ms)Running coprocessor post-close hooks at 1733294626501 (+3 ms)Closed at 1733294626501 2024-12-04T06:43:46,501 DEBUG [RS_CLOSE_META-regionserver/607fd5c6574c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T06:43:46,595 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(976): stopping server 607fd5c6574c,38257,1733294430862; all regions closed. 2024-12-04T06:43:46,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741834_1010 (size=25344) 2024-12-04T06:43:46,601 DEBUG [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/oldWALs 2024-12-04T06:43:46,601 INFO [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 607fd5c6574c%2C38257%2C1733294430862.meta:.meta(num 1733294433194) 2024-12-04T06:43:46,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741833_1009 (size=15856928) 2024-12-04T06:43:46,604 DEBUG [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/oldWALs 2024-12-04T06:43:46,604 INFO [RS:0;607fd5c6574c:38257 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 607fd5c6574c%2C38257%2C1733294430862:(num 1733294432862) 2024-12-04T06:43:46,604 DEBUG [RS:0;607fd5c6574c:38257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T06:43:46,604 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T06:43:46,605 INFO [RS:0;607fd5c6574c:38257 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T06:43:46,605 INFO [RS:0;607fd5c6574c:38257 {}] hbase.ChoreService(370): Chore service for: regionserver/607fd5c6574c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T06:43:46,605 INFO [RS:0;607fd5c6574c:38257 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T06:43:46,605 INFO [regionserver/607fd5c6574c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T06:43:46,606 INFO [RS:0;607fd5c6574c:38257 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38257 2024-12-04T06:43:46,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/607fd5c6574c,38257,1733294430862 2024-12-04T06:43:46,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T06:43:46,609 INFO [RS:0;607fd5c6574c:38257 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T06:43:46,614 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [607fd5c6574c,38257,1733294430862] 2024-12-04T06:43:46,616 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/607fd5c6574c,38257,1733294430862 already deleted, retry=false 2024-12-04T06:43:46,616 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 607fd5c6574c,38257,1733294430862 expired; onlineServers=0 2024-12-04T06:43:46,616 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '607fd5c6574c,45619,1733294430018' ***** 2024-12-04T06:43:46,616 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T06:43:46,616 INFO [M:0;607fd5c6574c:45619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T06:43:46,616 INFO [M:0;607fd5c6574c:45619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T06:43:46,616 DEBUG [M:0;607fd5c6574c:45619 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T06:43:46,616 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T06:43:46,616 DEBUG [M:0;607fd5c6574c:45619 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T06:43:46,616 DEBUG [master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.large.0-1733294432496 {}] cleaner.HFileCleaner(306): Exit Thread[master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.large.0-1733294432496,5,FailOnTimeoutGroup] 2024-12-04T06:43:46,616 DEBUG [master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.small.0-1733294432497 {}] cleaner.HFileCleaner(306): Exit Thread[master/607fd5c6574c:0:becomeActiveMaster-HFileCleaner.small.0-1733294432497,5,FailOnTimeoutGroup] 2024-12-04T06:43:46,617 INFO [M:0;607fd5c6574c:45619 {}] hbase.ChoreService(370): Chore service for: master/607fd5c6574c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T06:43:46,617 INFO [M:0;607fd5c6574c:45619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T06:43:46,617 DEBUG [M:0;607fd5c6574c:45619 {}] master.HMaster(1795): Stopping service threads 2024-12-04T06:43:46,617 INFO [M:0;607fd5c6574c:45619 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T06:43:46,617 INFO [M:0;607fd5c6574c:45619 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T06:43:46,617 ERROR [M:0;607fd5c6574c:45619 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38371,5,PEWorkerGroup] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1992759157) connection to localhost/127.0.0.1:38371 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:38371,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-04T06:43:46,618 INFO [M:0;607fd5c6574c:45619 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T06:43:46,618 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T06:43:46,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T06:43:46,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T06:43:46,619 DEBUG [M:0;607fd5c6574c:45619 {}] zookeeper.ZKUtil(347): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T06:43:46,619 WARN [M:0;607fd5c6574c:45619 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T06:43:46,619 INFO [M:0;607fd5c6574c:45619 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/.lastflushedseqids 2024-12-04T06:43:46,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742445_1621 (size=72) 2024-12-04T06:43:46,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T06:43:46,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38257-0x1017c33fe6b0001, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T06:43:46,715 INFO [RS:0;607fd5c6574c:38257 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T06:43:46,715 INFO [RS:0;607fd5c6574c:38257 {}] regionserver.HRegionServer(1031): Exiting; stopping=607fd5c6574c,38257,1733294430862; zookeeper connection closed. 2024-12-04T06:43:46,715 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1bd5661c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1bd5661c 2024-12-04T06:43:46,716 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T06:43:47,028 INFO [M:0;607fd5c6574c:45619 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T06:43:47,029 INFO [M:0;607fd5c6574c:45619 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T06:43:47,029 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T06:43:47,029 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:43:47,029 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:43:47,029 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T06:43:47,029 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:43:47,029 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=747.88 KB heapSize=914.09 KB 2024-12-04T06:43:47,044 DEBUG [M:0;607fd5c6574c:45619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddd6969910ce4a3a81b56081169510a2 is 82, key is hbase:meta,,1/info:regioninfo/1733294433281/Put/seqid=0 2024-12-04T06:43:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742446_1622 (size=5672) 2024-12-04T06:43:47,447 INFO [M:0;607fd5c6574c:45619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddd6969910ce4a3a81b56081169510a2 2024-12-04T06:43:47,467 DEBUG [M:0;607fd5c6574c:45619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6556d872b654525b39f9bcf4f8b4273 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733294600357/Put/seqid=0 2024-12-04T06:43:47,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742447_1623 (size=45703) 2024-12-04T06:43:47,871 INFO [M:0;607fd5c6574c:45619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.32 KB at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6556d872b654525b39f9bcf4f8b4273 2024-12-04T06:43:47,874 INFO [M:0;607fd5c6574c:45619 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6556d872b654525b39f9bcf4f8b4273 2024-12-04T06:43:47,889 DEBUG [M:0;607fd5c6574c:45619 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63d02ec6ed674332a5da4b794e9a6234 is 69, key is 607fd5c6574c,38257,1733294430862/rs:state/1733294432614/Put/seqid=0 2024-12-04T06:43:47,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073742448_1624 (size=5156) 2024-12-04T06:43:48,293 INFO [M:0;607fd5c6574c:45619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2036 (bloomFilter=true), to=hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63d02ec6ed674332a5da4b794e9a6234 2024-12-04T06:43:48,296 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddd6969910ce4a3a81b56081169510a2 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddd6969910ce4a3a81b56081169510a2 2024-12-04T06:43:48,299 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddd6969910ce4a3a81b56081169510a2, entries=8, sequenceid=2036, filesize=5.5 K 2024-12-04T06:43:48,299 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6556d872b654525b39f9bcf4f8b4273 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6556d872b654525b39f9bcf4f8b4273 2024-12-04T06:43:48,302 INFO [M:0;607fd5c6574c:45619 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6556d872b654525b39f9bcf4f8b4273 2024-12-04T06:43:48,302 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6556d872b654525b39f9bcf4f8b4273, entries=172, sequenceid=2036, filesize=44.6 K 2024-12-04T06:43:48,303 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/63d02ec6ed674332a5da4b794e9a6234 as hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/63d02ec6ed674332a5da4b794e9a6234 2024-12-04T06:43:48,305 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38371/user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/63d02ec6ed674332a5da4b794e9a6234, entries=1, sequenceid=2036, filesize=5.0 K 2024-12-04T06:43:48,306 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(3140): Finished flush of dataSize ~747.88 KB/765825, heapSize ~913.79 KB/935720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2036, compaction requested=false 2024-12-04T06:43:48,307 INFO [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T06:43:48,307 DEBUG [M:0;607fd5c6574c:45619 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733294627029Disabling compacts and flushes for region at 1733294627029Disabling writes for close at 1733294627029Obtaining lock to block concurrent updates at 1733294627029Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733294627029Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=765825, getHeapSize=935960, getOffHeapSize=0, getCellsCount=2233 at 1733294627029Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733294627030 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733294627030Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733294627044 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733294627044Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733294627451 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733294627467 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733294627467Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733294627875 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733294627889 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733294627889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e095d9: reopening flushed file at 1733294628296 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a5c3d63: reopening flushed file at 1733294628299 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457bb67b: reopening flushed file at 1733294628302 (+3 ms)Finished flush of dataSize ~747.88 KB/765825, heapSize ~913.79 KB/935720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1277ms, sequenceid=2036, compaction requested=false at 1733294628306 (+4 ms)Writing region close event to WAL at 1733294628307 (+1 ms)Closed at 1733294628307 2024-12-04T06:43:48,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741830_1006 (size=899372) 2024-12-04T06:43:48,310 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e6b4a3db-b540-34e6-bf62-71bb43a7b72d/MasterData/WALs/607fd5c6574c,45619,1733294430018/607fd5c6574c%2C45619%2C1733294430018.1733294431840 not finished, retry = 0 2024-12-04T06:43:48,411 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T06:43:48,411 INFO [M:0;607fd5c6574c:45619 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T06:43:48,411 INFO [M:0;607fd5c6574c:45619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45619 2024-12-04T06:43:48,411 INFO [M:0;607fd5c6574c:45619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T06:43:48,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T06:43:48,513 INFO [M:0;607fd5c6574c:45619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T06:43:48,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45619-0x1017c33fe6b0000, quorum=127.0.0.1:53418, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T06:43:48,517 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b60cd11{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T06:43:48,520 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286a1f36{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T06:43:48,520 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T06:43:48,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1118b2a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T06:43:48,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b40f5c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.log.dir/,STOPPED} 2024-12-04T06:43:48,524 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T06:43:48,524 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T06:43:48,524 WARN [BP-1546864157-172.17.0.2-1733294426706 heartbeating to localhost/127.0.0.1:38371 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T06:43:48,524 WARN [BP-1546864157-172.17.0.2-1733294426706 heartbeating to localhost/127.0.0.1:38371 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1546864157-172.17.0.2-1733294426706 (Datanode Uuid 3fd41660-1e1d-4ccd-869d-fa052efa4727) service to localhost/127.0.0.1:38371 2024-12-04T06:43:48,526 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data1/current/BP-1546864157-172.17.0.2-1733294426706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T06:43:48,527 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/cluster_b6f89d68-693b-011c-e262-9db36cad0a96/data/data2/current/BP-1546864157-172.17.0.2-1733294426706 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T06:43:48,527 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T06:43:48,535 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T06:43:48,536 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T06:43:48,536 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T06:43:48,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T06:43:48,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bdf09e12-6824-dd00-c7d1-9afe23d38e9c/hadoop.log.dir/,STOPPED} 2024-12-04T06:43:48,552 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T06:43:48,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down